From 99cd43b261e353a995f97e654a1f608830831d88 Mon Sep 17 00:00:00 2001 From: Perry Shafran Date: Mon, 30 Dec 2024 15:18:55 +0000 Subject: [PATCH 1/8] Adding the restart items to the ush directory --- ush/mesoscale/mesoscale_check_settings.py | 2 +- ush/mesoscale/mesoscale_create_output_dirs.py | 36 ++ ush/mesoscale/mesoscale_production_restart.py | 38 +- ...oscale_stats_grid2obs_create_job_script.py | 393 +++++++++++++++++- 4 files changed, 439 insertions(+), 30 deletions(-) diff --git a/ush/mesoscale/mesoscale_check_settings.py b/ush/mesoscale/mesoscale_check_settings.py index 11e3c14c4..042a5c1ca 100644 --- a/ush/mesoscale/mesoscale_check_settings.py +++ b/ush/mesoscale/mesoscale_check_settings.py @@ -58,7 +58,7 @@ 'MET_PLUS_CONF','MET_PLUS_OUT', 'LOG_MET_OUTPUT_TO_METPLUS','NEST','TEMP_DIR','GRID_DIR','URL_HEAD', ] -evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['bufr_ROOT'] +evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['RESTART_DIR', bufr_ROOT'] evs_mesoscale_settings_dict['RUN_GRID2OBS_PLOTS'] = [ 'MET_VERSION','IMG_HEADER','PRUNE_DIR','SAVE_DIR','LOG_TEMPLATE', 'LOG_LEVEL','STAT_OUTPUT_BASE_DIR','STAT_OUTPUT_BASE_TEMPLATE', diff --git a/ush/mesoscale/mesoscale_create_output_dirs.py b/ush/mesoscale/mesoscale_create_output_dirs.py index 393470bfe..1d83935f7 100644 --- a/ush/mesoscale/mesoscale_create_output_dirs.py +++ b/ush/mesoscale/mesoscale_create_output_dirs.py @@ -85,6 +85,7 @@ COMOUTplots = os.environ['COMOUTplots'] if STEP == 'stats': job_type = os.environ['job_type'] + RESTART_DIR = os.environ['RESTART_DIR'] if STEP == 'plots': RESTART_DIR = os.environ['RESTART_DIR'] @@ -230,23 +231,39 @@ working_output_base_dir = os.path.join( DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE ) + COMOUT_restart_base_dir = os.path.join( + RESTART_DIR, 'METplus_output', VERIF_TYPE + ) if job_type == 'generate': working_output_base_dir = os.path.join( DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE ) + COMOUT_restart_base_dir = os.path.join( + RESTART_DIR, 'METplus_output', VERIF_TYPE + ) if job_type == 'gather': working_output_base_dir = os.path.join( DATA, VERIF_CASE, 'METplus_output', 'gather_small' ) + COMOUT_restart_base_dir = os.path.join( + RESTART_DIR, 'METplus_output' + ) if job_type == 'gather2': working_output_base_dir = os.path.join( DATA, VERIF_CASE, 'METplus_output' ) + COMOUT_restart_base_dir = os.path.join( + RESTART_DIR, 'METplus_output' + ) if job_type == 'gather3': working_output_base_dir = os.path.join( DATA, VERIF_CASE, 'METplus_output' ) + COMOUT_restart_base_dir = os.path.join( + RESTART_DIR, 'METplus_output' + ) working_dir_list.append(working_output_base_dir) + COMOUT_dir_list.append(COMOUT_restart_base_dir) if job_type == 'reformat': working_dir_list.append(os.path.join( working_output_base_dir, NEST, 'pb2nc', 'confs' @@ -257,6 +274,9 @@ working_dir_list.append(os.path.join( working_output_base_dir, NEST, 'pb2nc', 'tmp' )) + COMOUT_dir_list.append(os.path.join( + COMOUT_restart_base_dir, NEST, 'pb2nc' + )) if NEST in ['spc_otlk', 'firewx']: working_dir_list.append(os.path.join( working_output_base_dir, 'genvxmask', 'confs' @@ -271,6 +291,10 @@ working_output_base_dir, 'genvxmask', NEST+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUT_restart_base_dir, 'genvxmask', + NEST+'.'+vdate_dt.strftime('%Y%m%d') + )) if job_type == 'generate': working_dir_list.append(os.path.join( working_output_base_dir, 'regrid_data_plane', 'confs' @@ -285,6 +309,10 @@ working_output_base_dir, 'regrid_data_plane', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUT_restart_base_dir, 'regrid_data_plane', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) working_dir_list.append(os.path.join( working_output_base_dir, 'point_stat', 'confs' )) @@ -298,6 +326,10 @@ working_output_base_dir, 'point_stat', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUT_restart_base_dir, 'point_stat', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) if job_type in ['gather', 'gather2', 'gather3']: working_dir_list.append(os.path.join( working_output_base_dir, 'stat_analysis', 'confs' @@ -312,6 +344,10 @@ working_output_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUT_restart_base_dir, 'stat_analysis', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) COMOUT_dir_list.append(os.path.join( COMOUTsmall, 'gather_small' diff --git a/ush/mesoscale/mesoscale_production_restart.py b/ush/mesoscale/mesoscale_production_restart.py index b19466cb3..3ed7a90da 100644 --- a/ush/mesoscale/mesoscale_production_restart.py +++ b/ush/mesoscale/mesoscale_production_restart.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -''' -Name: mesoscale_production_restart.py -Contact(s): Marcel Caron -Abstract: -''' +# ============================================================================= +# +# NAME: mesoscale_production_restart.py +# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# PURPOSE: Check the appropriate restart directory for restart files and copy +# the available files to the working directory +# +# ============================================================================= import os import glob @@ -20,20 +23,33 @@ NET = os.environ['NET'] RUN = os.environ['RUN'] COMPONENT = os.environ['COMPONENT'] -VERIF_CASE = os.environ['VERIF_CASE'] STEP = os.environ['STEP'] +VERIF_CASE = os.environ['VERIF_CASE'] # Copy files for restart -if STEP == 'plots': +if STEP == 'stats': + VERIF_CASE = os.environ['VERIF_CASE'] + RESTART_DIR = os.environ['RESTART_DIR'] + working_dir = os.path.join(DATA, VERIF_CASE) + completed_jobs_file = os.path.join(RESTART_DIR, 'completed_jobs.txt') + if os.path.exists(RESTART_DIR): + if (os.path.exists(completed_jobs_file) + and os.stat(completed_jobs_file).st_size != 0): + print(f"Copying restart directory {RESTART_DIR} " + +f"into working directory {working_dir}") + cutil.run_shell_command( + ['cp', '-rpv', RESTART_DIR, working_dir] + ) +elif STEP == 'plots': COMOUTplots = os.environ['COMOUTplots'] RESTART_DIR = os.environ['RESTART_DIR'] - COMPLETED_JOBS_FILE = os.environ['COMPLETED_JOBS_FILE'] - working_dir = os.path.join(DATA, VERIF_CASE, 'out') - completed_jobs_file = os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE) + SAVE_DIR = os.environ['SAVE_DIR'] + completed_jobs_file = os.path.join(RESTART_DIR, f'completed_jobs.txt') if os.path.exists(completed_jobs_file): if os.stat(completed_jobs_file).st_size != 0: cutil.run_shell_command( - ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), working_dir] + ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), SAVE_DIR] ) + print("END: "+os.path.basename(__file__)) diff --git a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py index 0c0147fcd..273d3dee0 100644 --- a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py +++ b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py @@ -35,14 +35,18 @@ MET_PATH = os.environ['MET_PATH'] MET_CONFIG = os.environ['MET_CONFIG'] DATA = os.environ['DATA'] +RESTART_DIR = os.environ['RESTART_DIR'] VDATE = os.environ['VDATE'] MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] +MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] metplus_launcher = 'run_metplus.py' machine_conf = os.path.join( os.environ['PARMevs'], 'metplus_config', 'machine.conf' ) +COMPLETED_JOBS_FILE = os.environ['COMPLETED_JOBS_FILE'] + if job_type == 'reformat': VERIF_TYPE = os.environ['VERIF_TYPE'] NEST = os.environ['NEST'] @@ -58,9 +62,6 @@ MIN_IHOUR = os.environ['MIN_IHOUR'] COMINobs = os.environ['COMINobs'] njob = os.environ['njob'] - MET_PLUS_OUT = os.path.join( - os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' - ) USHevs = os.environ['USHevs'] SKIP_IF_OUTPUT_EXISTS = os.environ['SKIP_IF_OUTPUT_EXISTS'] if NEST == 'spc_otlk': @@ -85,9 +86,6 @@ if NEST not in ['firewx', 'spc_otlk']: MASK_POLY_LIST = os.environ['MASK_POLY_LIST'] njob = os.environ['njob'] - MET_PLUS_OUT = os.path.join( - os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' - ) GRID = os.environ['GRID'] USHevs = os.environ['USHevs'] if NEST == 'spc_otlk': @@ -95,15 +93,9 @@ elif job_type == 'gather': VERIF_TYPE = os.environ['VERIF_TYPE'] njob = os.environ['njob'] - MET_PLUS_OUT = os.path.join( - os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' - ) elif job_type in ['gather2','gather3']: VERIF_TYPE = os.environ['VERIF_TYPE'] njob = os.environ['njob'] - MET_PLUS_OUT = os.path.join( - os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' - ) # Get expanded details from variable name if job_type == 'generate': @@ -366,62 +358,427 @@ pass elif STEP == 'stats': if job_type == 'reformat': + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list_iterative.append( + f'#jobs were restarted, and the following has already run successfully' + ) + job_cmd_list_iterative.append( + f'#{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'PB2NC_obs{VERIF_TYPE.upper()}.conf' + ) + job_cmd_list_iterative.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"pb2nc\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\"' + + ')\"' + ) + else: job_cmd_list_iterative.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'PB2NC_obs{VERIF_TYPE.upper()}.conf' ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"pb2nc\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) if job_type == 'generate': if FCST_VAR2_NAME: + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list_iterative.append( + f'#jobs were restarted, and the following has already run successfully' + ) + job_cmd_list_iterative.append( + f'#{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'PointStat_fcst{COMPONENT.upper()}_' + + f'obs{VERIF_TYPE.upper()}_{str(NEST).upper()}_VAR2.conf' + ) + job_cmd_list_iterative.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + else: job_cmd_list_iterative.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'PointStat_fcst{COMPONENT.upper()}_obs{VERIF_TYPE.upper()}_VAR2.conf' ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) else: if NEST == 'conusp': if VAR_NAME == 'PTYPE': + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list_iterative.append( + f'#jobs were restarted, and the following has already run successfully' + ) job_cmd_list_iterative.append( - f'{metplus_launcher} -c {machine_conf} ' + f'#{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'RegridDataPlane_fcst{COMPONENT.upper()}_PTYPE.conf' ) job_cmd_list_iterative.append( - f'python ' + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"regrid_data_plane\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + f'njob=\\\"{njob}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + f'#python ' + f'{USHevs}/{COMPONENT}/' + f'{COMPONENT}_{STEP}_{VERIF_CASE}_create_merged_ptype.py' ) - job_cmd_list_iterative.append( - f'{metplus_launcher} -c {machine_conf} ' + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"merged_ptype\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + f'njob=\\\"{njob}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + f'#{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'PointStat_fcst{COMPONENT.upper()}_' + f'obs{VERIF_TYPE.upper()}_{VAR_NAME}.conf' - ) + ) + job_cmd_list_iterative.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + else: + job_cmd_list_iterative.append( + f'{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'RegridDataPlane_fcst{COMPONENT.upper()}_PTYPE.conf' + ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"regrid_data_plane\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + f'njob=\\\"{njob}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + f'python ' + + f'{USHevs}/{COMPONENT}/' + + f'{COMPONENT}_{STEP}_{VERIF_CASE}_create_merged_ptype.py' + ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"merged_ptype\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + f'njob=\\\"{njob}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + f'{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'PointStat_fcst{COMPONENT.upper()}_' + + f'obs{VERIF_TYPE.upper()}_{VAR_NAME}.conf' + ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) + else: pstat_file_exist = cutil.check_pstat_files(job_env_vars_dict) if pstat_file_exist: print(f"skip this run, pstat already exist") else: + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list_iterative.append( + f'#jobs were restarted, and the following has already run successfully' + ) job_cmd_list_iterative.append( - f'{metplus_launcher} -c {machine_conf} ' + f'#{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'PointStat_fcst{COMPONENT.upper()}_obs{VERIF_TYPE.upper()}.conf' ) + job_cmd_list_iterative.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + else: + job_cmd_list_iterative.append( + f'{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'PointStat_fcst{COMPONENT.upper()}_' + + f'obs{VERIF_TYPE.upper()}.conf' + ) + job_cmd_list_iterative.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'vx_mask=\\\"${NEST}\\\", ' + + 'met_tool=\\\"point_stat\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'vhour=\\\"${VHOUR}\\\", ' + + 'fhr_start=\\\"${FHR_START}\\\", ' + + 'fhr_end=\\\"${FHR_END}\\\", ' + + 'fhr_incr=\\\"${FHR_INCR}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'var_name=\\\"${VAR_NAME}\\\"' + + ')\"' + ) + job_cmd_list_iterative.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) elif job_type == 'gather': + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list.append( + f'#jobs were restarted, and the following has already run successfully' + ) + job_cmd_list.append( + f'#{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'StatAnalysis_fcst{COMPONENT.upper()}_obs{VERIF_TYPE.upper()}' + + f'_GatherByDay.conf' + ) + job_cmd_list.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'met_tool=\\\"stat_analysis\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'net=\\\"${NET}\\\", ' + + 'step=\\\"${STEP}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'run=\\\"${RUN}\\\", ' + + f'job_type=\\\"{job_type}\\\"' + + ')\"' + ) + else: job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'StatAnalysis_fcst{COMPONENT.upper()}_obs{VERIF_TYPE.upper()}' + f'_GatherByDay.conf' ) + job_cmd_list.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + + 'met_tool=\\\"stat_analysis\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'net=\\\"${NET}\\\", ' + + 'step=\\\"${STEP}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'run=\\\"${RUN}\\\", ' + + f'job_type=\\\"{job_type}\\\"' + + ')\"' + ) + job_cmd_list.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) elif job_type == 'gather2': + if f'{job_type}_job{njob}' in cutil.get_completed_jobs(os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)): + job_cmd_list.append( + f'#jobs were restarted, and the following has already run successfully' + ) + job_cmd_list.append( + f'#{metplus_launcher} -c {machine_conf} ' + + f'-c {MET_PLUS_CONF}/' + + f'StatAnalysis_fcst{COMPONENT.upper()}' + + f'_GatherByCycle.conf' + ) + job_cmd_list.append( + f'#python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'met_tool=\\\"stat_analysis\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'net=\\\"${NET}\\\", ' + + 'step=\\\"${STEP}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'run=\\\"${RUN}\\\", ' + + 'vhr=\\\"${vhr}\\\", ' + + f'job_type=\\\"{job_type}\\\"' + + ')\"' + ) + else: job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' + f'StatAnalysis_fcst{COMPONENT.upper()}' + f'_GatherByCycle.conf' ) + job_cmd_list.append( + f'python -c ' + + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + 'verif_case=\\\"${VERIF_CASE}\\\", ' + + 'met_tool=\\\"stat_analysis\\\", ' + + 'vdate=\\\"${VDATE}\\\", ' + + 'net=\\\"${NET}\\\", ' + + 'step=\\\"${STEP}\\\", ' + + 'model=\\\"${MODELNAME}\\\", ' + + 'run=\\\"${RUN}\\\", ' + + 'vhr=\\\"${vhr}\\\", ' + + f'job_type=\\\"{job_type}\\\"' + + ')\"' + ) + job_cmd_list.append( + "python -c " + + f"'import mesoscale_util; mesoscale_util.mark_job_completed(" + + f"\"{os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)}\", " + + f"\"job{njob}\", job_type=\"{job_type}\")'" + ) elif job_type == 'gather3': job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' From f4fd8520d03a3d06e2f77f04a892ce58dc5b0e87 Mon Sep 17 00:00:00 2001 From: Perry Shafran Date: Mon, 30 Dec 2024 15:41:04 +0000 Subject: [PATCH 2/8] Updating evs_config files with the restart settings --- .../config.evs.prod.stats.mesoscale.atmos.grid2obs.nam | 4 ++++ .../config.evs.prod.stats.mesoscale.atmos.grid2obs.rap | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam index 585c1ea32..fafe82fc9 100644 --- a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam +++ b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam @@ -15,6 +15,10 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})" export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output export MET_CONFIG_OVERRIDES="" +# Restart Settings +export RESTART_DIR="${COMOUTsmall}/restart/c${vhr}" +export COMPLETED_JOBS_FILE="completed_jobs.txt" + # Time Settings export FHR_START=0 export FHR_INCR=6 diff --git a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap index 70cdfadf0..3947d8dc6 100644 --- a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap +++ b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap @@ -15,6 +15,10 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})" export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output export MET_CONFIG_OVERRIDES="" +# Restart Settings +export RESTART_DIR="${COMOUTsmall}/restart/c${vhr}" +export COMPLETED_JOBS_FILE="completed_jobs.txt" + # Time Settings export FHR_START=0 export FHR_INCR=1 From 29396a520cd9b638495730c2d552e1307efe2c34 Mon Sep 17 00:00:00 2001 From: Perry Shafran Date: Mon, 6 Jan 2025 14:31:56 +0000 Subject: [PATCH 3/8] More ush and ex-script updates --- .../exevs_mesoscale_nam_grid2obs_stats.sh | 9 +- ush/mesoscale/mesoscale_check_settings.py | 2 +- .../mesoscale_create_output_dirs_orig.py | 496 +++++++++ ush/mesoscale/mesoscale_util.py | 675 ++++++++++++- ush/mesoscale/mesoscale_util_save.py | 949 ++++++++++++++++++ 5 files changed, 2113 insertions(+), 18 deletions(-) create mode 100644 ush/mesoscale/mesoscale_create_output_dirs_orig.py create mode 100644 ush/mesoscale/mesoscale_util_save.py diff --git a/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh index a968bea86..38c11bd7e 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh @@ -42,6 +42,7 @@ echo "*****************************" # Reformat MET Data export job_type="reformat" export njob=1 + export run_restart=true for NEST in $NEST_LIST; do export NEST=$NEST for VERIF_TYPE in $VERIF_TYPES; do @@ -55,8 +56,12 @@ echo "*****************************" # Check for restart files reformat echo " Check for restart files reformat begin" if [ $evs_run_mode = production ]; then - ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh - export err=$?; err_chk + # Check For Restart Files + if [ "$run_restart" = true ]; then + python ${USHevs}/mesoscale/mesoscale_production_restart.py + export err=$?; err_chk + export run_restart=false + fi fi echo " Check for restart files reformat done" diff --git a/ush/mesoscale/mesoscale_check_settings.py b/ush/mesoscale/mesoscale_check_settings.py index 042a5c1ca..80473f4d8 100644 --- a/ush/mesoscale/mesoscale_check_settings.py +++ b/ush/mesoscale/mesoscale_check_settings.py @@ -58,7 +58,7 @@ 'MET_PLUS_CONF','MET_PLUS_OUT', 'LOG_MET_OUTPUT_TO_METPLUS','NEST','TEMP_DIR','GRID_DIR','URL_HEAD', ] -evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['RESTART_DIR', bufr_ROOT'] +evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['RESTART_DIR', 'bufr_ROOT'] evs_mesoscale_settings_dict['RUN_GRID2OBS_PLOTS'] = [ 'MET_VERSION','IMG_HEADER','PRUNE_DIR','SAVE_DIR','LOG_TEMPLATE', 'LOG_LEVEL','STAT_OUTPUT_BASE_DIR','STAT_OUTPUT_BASE_TEMPLATE', diff --git a/ush/mesoscale/mesoscale_create_output_dirs_orig.py b/ush/mesoscale/mesoscale_create_output_dirs_orig.py new file mode 100644 index 000000000..393470bfe --- /dev/null +++ b/ush/mesoscale/mesoscale_create_output_dirs_orig.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +# ============================================================================= +# +# NAME: mesoscale_create_output_dirs.py +# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# Roshan Shrestha, roshan.shrestha@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# PURPOSE: Define working/ output directories and create them if they don't +# exist. +# DEPENDENCIES: os.path.join([ +# SCRIPTSevs,COMPONENT,STEP, +# "_".join(["exevs",MODELNAME,VERIF_CASE,STEP+".sh"] +# )] +# +# ============================================================================= + +import os +import re +from datetime import datetime, timedelta as td +from mesoscale_plots_grid2obs_graphx_defs import graphics as graphics_g2o +from mesoscale_plots_precip_graphx_defs import graphics as graphics_pcp +from mesoscale_plots_headline_graphx_defs import graphics as graphics_hdl +from mesoscale_plots_snowfall_graphx_defs import graphics as graphics_sno +import mesoscale_util as cutil + +print(f"BEGIN: {os.path.basename(__file__)}") + +# Read in environment variables +evs_ver = os.environ['evs_ver'] +EVSIN = os.environ['EVSIN'] +COMOUT = os.environ['COMOUT'] +DATA = os.environ['DATA'] +NET = os.environ['NET'] +RUN = os.environ['RUN'] +COMPONENT = os.environ['COMPONENT'] +VERIF_CASE = os.environ['VERIF_CASE'] +STEP = os.environ['STEP'] +MODELNAME = os.environ['MODELNAME'] +VDATE = os.environ['VDATE'] +vdate_dt = datetime.strptime(VDATE, '%Y%m%d') +if VERIF_CASE == "precip": + if STEP == 'prep': + FHR_END_FULL = os.environ['FHR_END_FULL'] + FHR_END_SHORT = os.environ['FHR_END_SHORT'] + fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) + start_date_dt = vdate_dt - td(hours=fhr_end_max) + VERIF_TYPE = os.environ['VERIF_TYPE'] + OBSNAME = os.environ['OBSNAME'] + elif STEP == 'stats': + FHR_END_FULL = os.environ['FHR_END_FULL'] + FHR_END_SHORT = os.environ['FHR_END_SHORT'] + fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) + start_date_dt = vdate_dt - td(hours=fhr_end_max) + VERIF_TYPE = os.environ['VERIF_TYPE'] + OBSNAME = os.environ['OBSNAME'] + COMOUTsmall = os.environ['COMOUTsmall'] + elif STEP == 'plots': + all_eval_periods = cutil.get_all_eval_periods(graphics_pcp) + COMOUTplots = os.environ['COMOUTplots'] +elif VERIF_CASE == "grid2obs": + if STEP == 'prep': + NEST = os.environ['NEST'] + if STEP == 'stats': + NEST = os.environ['NEST'] + FHR_END_FULL = os.environ['FHR_END_FULL'] + FHR_END_SHORT = os.environ['FHR_END_SHORT'] + fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) + start_date_dt = vdate_dt - td(hours=fhr_end_max) + VERIF_TYPE = os.environ['VERIF_TYPE'] + OBSNAME = os.environ['OBSNAME'] + COMOUTsmall = os.environ['COMOUTsmall'] + elif STEP == 'plots': + all_eval_periods = cutil.get_all_eval_periods(graphics_g2o) + COMOUTplots = os.environ['COMOUTplots'] +elif VERIF_CASE == "headline": + if STEP == 'plots': + all_eval_periods = cutil.get_all_eval_periods(graphics_hdl) + COMOUTplots = os.environ['COMOUTplots'] +elif VERIF_CASE == "snowfall": + if STEP == 'prep': + pass + elif STEP == 'stats': + pass + elif STEP == 'plots': + all_eval_periods = cutil.get_all_eval_periods(graphics_sno) + COMOUTplots = os.environ['COMOUTplots'] +if STEP == 'stats': + job_type = os.environ['job_type'] +if STEP == 'plots': + RESTART_DIR = os.environ['RESTART_DIR'] + + +# Define data base directorie +data_base_dir = os.path.join(DATA, VERIF_CASE, 'data') +data_dir_list = [data_base_dir] +if VERIF_CASE == 'precip': + if STEP == 'prep': + data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) + data_dir_list.append(os.path.join(data_base_dir, OBSNAME)) + if STEP == 'stats': + data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) + data_dir_list.append(os.path.join(data_base_dir, OBSNAME)) +elif VERIF_CASE == 'grid2obs': + if STEP == 'stats': + data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) + data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'merged_ptype')) + data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'tmp')) + data_dir_list.append(os.path.join( + data_base_dir, OBSNAME, 'prepbufr' + )) +elif VERIF_CASE == 'snowfall': + if STEP == 'stats': + pass + +# Create data directories and subdirectories +for data_dir in data_dir_list: + if not os.path.exists(data_dir): + print(f"Creating data directory: {data_dir}") + os.makedirs(data_dir, mode=0o755) + +# Create job script base directory +job_scripts_dirs = [] +if STEP == 'prep': + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, STEP, 'prep_job_scripts')) +if STEP == 'stats': + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'reformat')) + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'generate')) + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather')) + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather2')) + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather3')) +if STEP == 'plots': + job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, STEP, 'plotting_job_scripts')) +for job_scripts_dir in job_scripts_dirs: + if not os.path.exists(job_scripts_dir): + print(f"Creating job script directory: {job_scripts_dir}") + os.makedirs(job_scripts_dir, mode=0o755) + +# Define working and COMOUT directories +working_dir_list = [] +COMOUT_dir_list = [] +if STEP == 'prep': + if VERIF_CASE == 'grid2obs': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data', + NEST+'.'+vdate_dt.strftime('%Y%m%d') + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + NEST+'.'+vdate_dt.strftime('%Y%m%d') + )) +elif STEP == 'stats': + if VERIF_CASE == 'precip': + if job_type == 'reformat': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE + ) + if job_type == 'generate': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE + ) + if job_type == 'gather': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', 'gather_small' + ) + if job_type == 'gather2': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output' + ) + if job_type == 'gather3': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output' + ) + working_dir_list.append(working_output_base_dir) + if job_type == 'reformat': + working_dir_list.append(os.path.join( + working_output_base_dir, 'pcp_combine', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'pcp_combine', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'pcp_combine', 'tmp' + )) + if job_type == 'generate': + working_dir_list.append(os.path.join( + working_output_base_dir, 'grid_stat', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'grid_stat', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'grid_stat', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'grid_stat', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + if job_type in ['gather', 'gather2', 'gather3']: + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + if job_type == 'reformat': + working_dir_list.append(os.path.join( + working_output_base_dir, 'pcp_combine', + OBSNAME+'.'+date_dt.strftime('%Y%m%d') + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'pcp_combine', + MODELNAME+'.'+date_dt.strftime('init%Y%m%d') + )) + elif VERIF_CASE == "grid2obs": + if job_type == 'reformat': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE + ) + if job_type == 'generate': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE + ) + if job_type == 'gather': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output', 'gather_small' + ) + if job_type == 'gather2': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output' + ) + if job_type == 'gather3': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE, 'METplus_output' + ) + working_dir_list.append(working_output_base_dir) + if job_type == 'reformat': + working_dir_list.append(os.path.join( + working_output_base_dir, NEST, 'pb2nc', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, NEST, 'pb2nc', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, NEST, 'pb2nc', 'tmp' + )) + if NEST in ['spc_otlk', 'firewx']: + working_dir_list.append(os.path.join( + working_output_base_dir, 'genvxmask', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'genvxmask', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'genvxmask', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'genvxmask', + NEST+'.'+vdate_dt.strftime('%Y%m%d') + )) + if job_type == 'generate': + working_dir_list.append(os.path.join( + working_output_base_dir, 'regrid_data_plane', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'regrid_data_plane', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'regrid_data_plane', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'regrid_data_plane', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'point_stat', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'point_stat', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'point_stat', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'point_stat', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + if job_type in ['gather', 'gather2', 'gather3']: + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'confs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'logs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', 'tmp' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'stat_analysis', + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + COMOUT_dir_list.append(os.path.join( + COMOUTsmall, + 'gather_small' + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + if job_type == 'reformat': + working_dir_list.append(os.path.join( + working_output_base_dir, NEST, 'pb2nc', + OBSNAME+'.'+vdate_dt.strftime('%Y%m%d') + )) + working_dir_list.append(os.path.join( + working_output_base_dir, NEST, 'pb2nc', + MODELNAME+'.'+vdate_dt.strftime('init%Y%m%d') + )) + elif VERIF_CASE == "snowfall": + pass + working_dir_list.append(os.path.join( + DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type + )) +elif STEP == 'plots': + if VERIF_CASE == 'grid2obs': + + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'logs' + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, VERIF_CASE + )) + for plot_group in ['cape', 'ceil_vis','sfc_upper']: + for eval_period in all_eval_periods: + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', str(plot_group).lower(), + str(eval_period).lower() + )) + COMOUT_dir_list.append(os.path.join( + RESTART_DIR, str(plot_group).lower(), + str(eval_period).lower() + )) + if VERIF_CASE == 'precip': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'logs' + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, VERIF_CASE + )) + for plot_group in ['precip']: + for eval_period in all_eval_periods: + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', str(plot_group).lower(), + str(eval_period).lower() + )) + if not str(eval_period).lower() == 'na' : + COMOUT_dir_list.append(os.path.join( + RESTART_DIR, str(plot_group).lower(), + str(eval_period).lower() + )) + elif VERIF_CASE == 'snowfall': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'logs' + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, VERIF_CASE + )) + for plot_group in ['precip']: + for eval_period in all_eval_periods: + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', str(plot_group).lower(), + str(eval_period).lower() + )) + COMOUT_dir_list.append(os.path.join( + RESTART_DIR, str(plot_group).lower(), + str(eval_period).lower() + )) + elif VERIF_CASE == 'headline': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'logs' + )) + COMOUT_dir_list.append(os.path.join( + COMOUT, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, + )) + COMOUT_dir_list.append(os.path.join( + COMOUTplots, VERIF_CASE + )) + for plot_group in ['sfc_upper']: + for eval_period in all_eval_periods: + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', str(plot_group).lower(), + str(eval_period).lower() + )) + COMOUT_dir_list.append(os.path.join( + RESTART_DIR, str(plot_group).lower(), + str(eval_period).lower() + )) +# Create working output and COMOUT directories +for working_dir in working_dir_list: + if not os.path.exists(working_dir): + print(f"Creating working output directory: {working_dir}") + os.makedirs(working_dir, mode=0o755, exist_ok=True) + else: + print(f"Tried creating working output directory but already exists: {working_dir}") +for COMOUT_dir in COMOUT_dir_list: + if not os.path.exists(COMOUT_dir): + print(f"Creating COMOUT directory: {COMOUT_dir}") + os.makedirs(COMOUT_dir, mode=0o755, exist_ok=True) + +print(f"END: {os.path.basename(__file__)}") diff --git a/ush/mesoscale/mesoscale_util.py b/ush/mesoscale/mesoscale_util.py index 570eee8e4..c7acebc0b 100644 --- a/ush/mesoscale/mesoscale_util.py +++ b/ush/mesoscale/mesoscale_util.py @@ -372,6 +372,284 @@ def format_filler(unfilled_file_format, valid_time_dt, init_time_dt, filled_file_format_chunk) return filled_file_format +def get_completed_jobs(completed_jobs_file): + completed_jobs = set() + if os.path.exists(completed_jobs_file): + with open(completed_jobs_file, 'r') as f: + completed_jobs = set(f.read().splitlines()) + return completed_jobs + +def mark_job_completed(completed_jobs_file, job_name, job_type=""): + with open(completed_jobs_file, 'a') as f: + if job_type: + f.write(job_type + "_" + job_name + "\n") + else: + f.write(job_name + "\n") + +def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, + run=None, step=None, model=None, vdate=None, vhr=None, + verif_case=None, verif_type=None, vx_mask=None, + job_type=None, var_name=None, vhour=None, + fhr_start=None, fhr_end=None, fhr_incr=None, + njob=None, acc=None, nbrhd=None): + sub_dirs = [] + copy_files = [] + if met_tool == "ascii2nc": + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + vx_mask, + met_tool, + )) + copy_files.append(f'{verif_type}.{vdate}{vhour}.nc') + elif met_tool == 'genvxmask': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{vx_mask}.{vdate}', + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + copy_files.append(f'{vx_mask}_t{vhour}z_f{str(fhr).zfill(2)}.nc') + elif met_tool == 'grid_stat': + if verif_case == "snowfall": + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, var_name, + acc, nbrhd + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + copy_files.append( + f'{met_tool}_{model}_{var_name}*{acc}H_{str(verif_type).upper()}_NBRHD{nbrhd}*_' + + f'{str(fhr).zfill(2)}0000L_{vdate}_{vhour}0000V.stat' + ) + else: + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, acc, nbrhd + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + copy_files.append( + f'{met_tool}_{model}_*_{acc}H_{str(verif_type).upper()}_NBRHD{nbrhd}*_' + + f'{str(fhr).zfill(2)}0000L_{vdate}_{vhour}0000V.stat' + ) + elif met_tool == 'merged_ptype': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, njob + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'data', + model, + met_tool, + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') + idt = vdt - td(hours=int(fhr)) + idate = idt.strftime('%Y%m%d') + ihour = idt.strftime('%H') + copy_files.append( + f'{met_tool}_{verif_type}_{vx_mask}_job{njob}_' + + f'init{idate}{ihour}_fhr{str(fhr).zfill(2)}.nc' + ) + elif met_tool == 'pb2nc': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + vx_mask, + met_tool, + )) + copy_files.append(f'prepbufr.*.{vdate}{vhour}.nc') + elif met_tool == 'pcp_combine': + if verif_case == "snowfall": + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, var_name, acc + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') + idt = vdt - td(hours=int(fhr)) + idate = idt.strftime('%Y%m%d') + ihour = idt.strftime('%H') + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + )) + copy_files.append( + f'{model}.{var_name}.init{idate}.t{ihour}z.f{str(fhr).zfill(3)}.a{acc}h.{vx_mask}.nc' + ) + else: + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, acc + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') + idt = vdt - td(hours=int(fhr)) + idate = idt.strftime('%Y%m%d') + ihour = idt.strftime('%H') + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + )) + copy_files.append( + f'{model}.init{idate}.t{ihour}z.f{str(fhr).zfill(3)}.a{acc}h.{vx_mask}.nc' + ) + elif met_tool == 'point_stat': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, var_name + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + copy_files.append( + f'{met_tool}_{model}_{vx_mask}_{var_name}_OBS*_{str(fhr).zfill(2)}0000L_{vdate}_' + + f'{vhour}0000V.stat' + ) + elif met_tool == 'regrid_data_plane': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, vx_mask, met_tool, + vdate, vhour, fhr_start, fhr_end, fhr_incr, model, njob + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) + for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + copy_files.append( + f'{met_tool}_{model}_t{vhour}z_{verif_type}_{vx_mask}_job{njob}_' + + f'fhr{str(fhr).zfill(2)}.nc' + ) + elif met_tool == 'stat_analysis': + if job_type == 'gather': + check_if_none = [ + data_dir, restart_dir, verif_case, verif_type, met_tool, vdate, + net, step, model, run + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + 'gather_small', + met_tool, + f'{model}.{vdate}' + )) + copy_files.append( + f'{net}.{step}.{model}.{run}.{verif_case}.{verif_type}' + + f'.v{vdate}.stat' + ) + elif job_type == 'gather2': + check_if_none = [ + data_dir, restart_dir, verif_case, met_tool, vdate, net, step, + model, run, vhr + ] + if any([var is None for var in check_if_none]): + e = (f"FATAL ERROR: None encountered as an argument while copying" + + f" {met_tool} METplus output to COMOUT directory.") + raise TypeError(e) + sub_dirs.append(os.path.join( + 'METplus_output', + met_tool, + f'{model}.{vdate}' + )) + copy_files.append( + f'{net}.{step}.{model}.{run}.{verif_case}.v{vdate}.c{vhr}z.stat' + ) + for sub_dir in sub_dirs: + for copy_file in copy_files: + origin_path = os.path.join( + data_dir, verif_case, sub_dir, copy_file + ) + dest_path = os.path.join(restart_dir, sub_dir) + if not glob.glob(origin_path): + continue + if not os.path.exists(dest_path): + print(f"FATAL ERROR: Could not copy METplus output to COMOUT directory" + + f" {dest_path} because the path does not already exist.") + continue + if len(glob.glob(origin_path)) == len(glob.glob(os.path.join(dest_path, copy_file))): + print(f"Not copying restart files to restart_directory" + + f" {dest_path} because they already exist.") + else: + run_shell_command( + ['cp', '-rpv', origin_path, os.path.join(dest_path,'.')] + ) + def initalize_job_env_dict(): """! This initializes a dictionary of environment variables and their values to be set for the job pulling from environment variables @@ -386,8 +664,8 @@ def initalize_job_env_dict(): 'machine', 'evs_ver', 'HOMEevs', 'FIXevs', 'USHevs', 'DATA', 'NET', 'RUN', 'VERIF_CASE', 'STEP', 'COMPONENT', 'evs_run_mode', 'COMROOT', 'COMIN', 'COMOUT', 'COMOUTsmall', 'COMOUTfinal', 'EVSIN', - 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', - 'MET_ROOT', + 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', + 'MET_ROOT', 'MET_TMP_DIR', 'MODELNAME', 'JOB_GROUP' ] job_env_dict = {} @@ -827,19 +1105,6 @@ def snowfall_check_model_input_output_files(job_dict): return (all_input_file_exist, input_files_list, \ all_COMOUT_file_exist, COMOUT_files_list, DATA_files_list) - - -def get_completed_jobs(completed_jobs_file): - completed_jobs = set() - if os.path.exists(completed_jobs_file): - with open(completed_jobs_file, 'r') as f: - completed_jobs = set(f.read().splitlines()) - return completed_jobs - -def mark_job_completed(completed_jobs_file, job_name): - with open(completed_jobs_file, 'a') as f: - f.write(job_name + "\n") - # Construct a file name given a template def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", VDATE="YYYYmmdd", VHOUR="HH", VDATEHOUR="YYYYmmddHH", @@ -947,3 +1212,383 @@ def preprocess_prepbufr(indir, fname, workdir, outdir, subsets): ) os.chdir(wd) +# Create a list of ccpa file paths +def get_ccpa_qpe_templates(indir, vdates, obs_acc, target_acc, nest, paths=[]): + ''' + indir..... - (str) Input directory for prepbufr file data + vdates.... - (datetime object) List of datetimes used to fill templates + obs_acc... - (str) precip accumulation interval of ccpa files in hours + target_acc - (str) target precip accumulation interval of combined + ccpa files in hours + nest...... - (str) domain used to find ccpa files + paths..... - (list of str) list of paths to append the prepbufr paths to + Default is empty. + ''' + ccpa_paths = [] + for v, vdate in enumerate(vdates): + vh = vdate.strftime('%H') + vd = vdate.strftime('%Y%m%d') + if int(target_acc) == 1: + if int(obs_acc) == 1: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + elif int(target_acc) == 3: + if int(obs_acc) == 1: + offsets = [0, 1, 2] + elif int(obs_acc) == 3: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + elif int(target_acc) == 24: + if int(obs_acc) == 1: + offsets = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18 , 19, 20, + 21, 22, 23 + ] + elif int(obs_acc) == 3: + offsets = [0, 3, 6, 9, 12, 15, 18, 21] + elif int(obs_acc) == 24: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + else: + raise ValueError(f"target_acc is not valid: \"{target_acc}\"") + for offset in offsets: + use_vdate = vdate - td(hours=int(offset)) + use_vd = use_vdate.strftime('%Y%m%d') + use_vh = use_vdate.strftime('%H') + template = os.path.join( + indir, + 'ccpa.{VDATE}', + 'ccpa.t{VHOUR}z.' + f'{obs_acc}h.hrap.{nest}.gb2' + ) + ccpa_paths.append(fname_constructor( + template, VDATE=use_vd, VHOUR=use_vh + )) + return np.concatenate((paths, np.unique(ccpa_paths))) + +# Create a list of mrms file paths +def get_mrms_qpe_templates(indir, vdates, obs_acc, target_acc, nest, paths=[]): + ''' + indir..... - (str) Input directory for prepbufr file data + vdates.... - (datetime object) List of datetimes used to fill templates + obs_acc... - (str) precip accumulation interval of mrms files in hours + target_acc - (str) target precip accumulation interval of combined + mrms files in hours + nest...... - (str) domain used to find mrms files + paths..... - (list of str) list of paths to append the prepbufr paths to + Default is empty. + ''' + mrms_paths = [] + for v, vdate in enumerate(vdates): + vh = vdate.strftime('%H') + vd = vdate.strftime('%Y%m%d') + if int(target_acc) == 1: + if int(obs_acc) == 1: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + elif int(target_acc) == 3: + if int(obs_acc) == 1: + offsets = [0, 1, 2] + elif int(obs_acc) == 3: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + elif int(target_acc) == 24: + if int(obs_acc) == 1: + offsets = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18 , 19, 20, + 21, 22, 23 + ] + elif int(obs_acc) == 3: + offsets = [0, 3, 6, 9, 12, 15, 18, 21] + elif int(obs_acc) == 24: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + else: + raise ValueError(f"target_acc is not valid: \"{target_acc}\"") + for offset in offsets: + use_vdate = vdate - td(hours=int(offset)) + use_vd = use_vdate.strftime('%Y%m%d') + use_vh = use_vdate.strftime('%H') + template = os.path.join( + indir, + 'mrms.{VDATE}', + 'mrms.t{VHOUR}z.' + f'{obs_acc}h.{nest}.gb2' + ) + mrms_paths.append(fname_constructor( + template, VDATE=use_vd, VHOUR=use_vh + )) + return np.concatenate((paths, np.unique(mrms_paths))) + +# Create a list of nohrsc file paths +def get_nohrsc_qpe_templates(indir, vdates, obs_acc, target_acc, nest, paths=[]): + ''' + indir..... - (str) Input directory for nohrsc file data + vdates.... - (datetime object) List of datetimes used to fill templates + obs_acc... - (str) snow accumulation interval of nohrsc files in hours + target_acc - (str) target snow accumulation interval of combined + nohrsc files in hours + nest...... - (str) domain used to find nohrsc files + paths..... - (list of str) list of paths to append the nohrsc paths to + Default is empty. + ''' + nohrsc_paths = [] + for v, vdate in enumerate(vdates): + vh = vdate.strftime('%H') + vd = vdate.strftime('%Y%m%d') + if int(target_acc) == 6: + if int(obs_acc) == 6: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + elif int(target_acc) == 24: + if int(obs_acc) == 6: + offsets = [0, 6, 12, 18] + elif int(obs_acc) == 24: + offsets = [0] + else: + raise ValueError(f"obs_acc is not valid: \"{obs_acc}\"") + else: + raise ValueError(f"target_acc is not valid: \"{target_acc}\"") + for offset in offsets: + use_vdate = vdate - td(hours=int(offset)) + use_vd = use_vdate.strftime('%Y%m%d') + use_vh = use_vdate.strftime('%H') + template = os.path.join( + indir, + '{VDATE}', 'wgrbbul', 'nohrsc_snowfall', + f'sfav2_CONUS_{int(obs_acc)}h_' + '{VDATE}{VHOUR}_grid184.grb2' + ) + nohrsc_paths.append(fname_constructor( + template, VDATE=use_vd, VHOUR=use_vh + )) + return np.concatenate((paths, np.unique(nohrsc_paths))) + +# Return a list of missing ccpa files needed to create "target_acc" +def check_ccpa_files(indir, vdate, obs_acc, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + obs_acc... - (str) precip accumulation interval of ccpa files in hours + target_acc - (str) target precip accumulation interval of combined + ccpa files in hours + nest...... - (str) domain used to find ccpa files + ''' + paths = get_ccpa_qpe_templates(indir, [vdate], obs_acc, target_acc, nest) + return [path for path in paths if not os.path.exists(path)] + +# Return a list of missing mrms files needed to create "target_acc" +def check_mrms_files(indir, vdate, obs_acc, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + obs_acc... - (str) precip accumulation interval of mrms files in hours + target_acc - (str) target precip accumulation interval of combined + mrms files in hours + nest...... - (str) domain used to find mrms files + ''' + paths = get_mrms_qpe_templates(indir, [vdate], obs_acc, target_acc, nest) + return [path for path in paths if not os.path.exists(path)] + +# Return a list of missing nohrsc files needed to create "target_acc" +def check_nohrsc_files(indir, vdate, obs_acc, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + obs_acc... - (str) precip accumulation interval of nohrsc files in hours + target_acc - (str) target precip accumulation interval of combined + nohrsc files in hours + nest...... - (str) domain used to find nohrsc files + ''' + paths = get_nohrsc_qpe_templates(indir, [vdate], obs_acc, target_acc, nest) + return [path for path in paths if not os.path.exists(path)] + +# Return the obs accumulation interval needed to create target_acc, based on +# available ccpa files +def get_ccpa_accums(indir, vdate, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + target_acc - (str) target precip accumulation interval of combined + ccpa files in hours + nest...... - (str) domain used to find ccpa files + ''' + if int(target_acc) == 1: + # check 1-h obs + obs_acc = "01" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + return None + else: + return obs_acc + elif int(target_acc) == 3: + # check 3-h obs + obs_acc = "03" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + # check 1-h obs + obs_acc = "01" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + return None + else: + return obs_acc + else: + return obs_acc + elif int(target_acc) == 24: + + # check 24-h obs + obs_acc = "24" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + # check 3-h obs + obs_acc = "03" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + # check 1-h obs + obs_acc = "01" + missing_ccpa = check_ccpa_files(indir, vdate, obs_acc, target_acc, nest) + if missing_ccpa: + return None + else: + return obs_acc + else: + return obs_acc + else: + return obs_acc + else: + raise ValueError(f"Invalid target_acc: \"{target_acc}\"") + +# Return the obs accumulation interval needed to create target_acc, based on +# available mrms files +def get_mrms_accums(indir, vdate, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + target_acc - (str) target precip accumulation interval of combined + mrms files in hours + nest...... - (str) domain used to find mrms files + ''' + if int(target_acc) == 1: + # check 1-h obs + obs_acc = "01" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + return None + else: + return obs_acc + elif int(target_acc) == 3: + # check 3-h obs + obs_acc = "03" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + # check 1-h obs + obs_acc = "01" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + return None + else: + return obs_acc + else: + return obs_acc + elif int(target_acc) == 24: + + # check 24-h obs + obs_acc = "24" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + # check 3-h obs + obs_acc = "03" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + # check 1-h obs + obs_acc = "01" + missing_mrms = check_mrms_files(indir, vdate, obs_acc, target_acc, nest) + if missing_mrms: + return None + else: + return obs_acc + else: + return obs_acc + else: + return obs_acc + else: + raise ValueError(f"Invalid target_acc: \"{target_acc}\"") + +# Return the obs accumulation interval needed to create target_acc, based on +# available nohrsc files +def get_nohrsc_accums(indir, vdate, target_acc, nest): + ''' + indir..... - (str) Input directory for prepbufr file data + vdate..... - (datetime object) datetime used to fill templates + target_acc - (str) target precip accumulation interval of combined + nohrsc files in hours + nest...... - (str) domain used to find nohrsc files + ''' + if int(target_acc) == 6: + # check 6-h obs + obs_acc = "06" + missing_nohrsc = check_nohrsc_files(indir, vdate, obs_acc, target_acc, nest) + if missing_nohrsc: + return None + else: + return obs_acc + elif int(target_acc) == 24: + # check 24-h obs + obs_acc = "24" + missing_nohrsc = check_nohrsc_files(indir, vdate, obs_acc, target_acc, nest) + if missing_nohrsc: + # check 6-h obs + obs_acc = "06" + missing_nohrsc = check_nohrsc_files(indir, vdate, obs_acc, target_acc, nest) + if missing_nohrsc: + return None + else: + return obs_acc + else: + return obs_acc + else: + raise ValueError(f"Invalid target_acc: \"{target_acc}\"") + +# Return the obs accumulation interval needed to create target_acc, based on +# available input files +def get_obs_accums(indir, vdate, target_acc, nest, obsname, job_type='reformat'): + ''' + indir..... - (str) Input directory for obs file data + vdate..... - (datetime object) datetime used to fill templates + target_acc - (str) target precip accumulation interval of combined + obs files in hours + nest...... - (str) domain used to find obs files + obsname... - (str) name of input file dataset + ''' + if obsname == "mrms": + return get_mrms_accums(indir, vdate, target_acc, nest) + elif obsname == "ccpa": + return get_ccpa_accums(indir, vdate, target_acc, nest) + elif obsname == "nohrsc": + return get_nohrsc_accums(indir, vdate, target_acc, nest) + else: + raise ValueError(f"Invalid obsname: \"{obsname}\"") + +# Return availability of obs needed for job +def get_obs_avail(indir, vdate, nest, obsname): + ''' + indir..... - (str) Input directory for obs file data + vdate..... - (datetime object) datetime used to fill templates + nest...... - (str) domain used to find obs files + obsname... - (str) name of input file dataset + ''' + if obsname in ["raob", "metar"]: + paths = get_prepbufr_templates(indir, [vdate], obsname=obsname, already_preprocessed=True) + if paths.size > 0: + return all([os.path.exists(fname) for fname in paths]) + else: + return False + else: + raise ValueError(f"Invalid obsname: \"{obsname}\"") diff --git a/ush/mesoscale/mesoscale_util_save.py b/ush/mesoscale/mesoscale_util_save.py new file mode 100644 index 000000000..570eee8e4 --- /dev/null +++ b/ush/mesoscale/mesoscale_util_save.py @@ -0,0 +1,949 @@ +#!/usr/bin/env python3 +# ============================================================================= +# +# NAME: mesoscale_util.py +# CONTRIBUTOR(S): RS, roshan.shrestha@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# Mallory Row, mallory.row@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# PURPOSE: Various Utilities for EVS MESOSCALE Verification +# +# ============================================================================= + +import os +import sys +from datetime import datetime, timedelta as td +import numpy as np +import glob +import subprocess +from collections.abc import Iterable + +def flatten(xs): + for x in xs: + if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + yield from flatten(x) + else: + yield x + +def get_data_type(fname): + data_type_dict = { + 'PrepBUFR': { + 'and':[''], + 'or':['prepbufr'], + 'not':[], + 'type': 'anl' + }, + 'SPC Outlook Area': { + 'and':[''], + 'or':['spc_otlk'], + 'not':[], + 'type': 'gen' + }, + 'NAM': { + 'and':[''], + 'or':['nam'], + 'not':[], + 'type': 'fcst' + }, + 'RAP': { + 'and':[''], + 'or':['rap'], + 'not':[], + 'type': 'fcst' + }, + } + for k in data_type_dict: + if not data_type_dict[k]['and'] or not any(data_type_dict[k]['and']): + data_type_dict[k]['and'] = [''] + if not data_type_dict[k]['or'] or not any(data_type_dict[k]['or']): + data_type_dict[k]['or'] = [''] + if not data_type_dict[k]['not'] or not any(data_type_dict[k]['not']): + data_type_dict[k]['not'] = [] + data_names = [ + k for k in data_type_dict + if ( + all(map(fname.__contains__, data_type_dict[k]['and'])) + and any(map(fname.__contains__, data_type_dict[k]['or'])) + and not any(map(fname.__contains__, data_type_dict[k]['not'])) + ) + ] + if len(data_names) == 1: + data_name = data_names[0] + return data_name, data_type_dict[data_name]['type'] + else: + data_name = "Unknown" + return data_name, 'unk' + +def get_all_eval_periods(graphics): + all_eval_periods = [] + for component in graphics: + for verif_case in graphics[component]: + for verif_type in graphics[component][verif_case]: + verif_type_dict = graphics[component][verif_case][verif_type] + for models in verif_type_dict: + for plot_type in verif_type_dict[models]: + all_eval_periods.append( + verif_type_dict[models][plot_type]['EVAL_PERIODS'] + ) + return np.unique(np.hstack(all_eval_periods)) + +def get_fhr_start(vhour, acc, fhr_incr, min_ihour): + fhr_start = ( + float(vhour) + float(min_ihour) + + ( + float(fhr_incr) + * np.ceil( + (float(acc)-float(vhour)-float(min_ihour)) + / float(fhr_incr) + ) + ) + ) + return int(fhr_start) + +def run_shell_command(command, capture_output=False): + """! Run shell command + + Args: + command - list of argument entries (string) + + Returns: + + """ + print("Running "+' '.join(command)) + if any(mark in ' '.join(command) for mark in ['"', "'", '|', '*', '>']): + run_command = subprocess.run( + ' '.join(command), shell=True, capture_output=capture_output + ) + else: + run_command = subprocess.run(command, capture_output=capture_output) + if run_command.returncode != 0: + print("ERROR: "+''.join(run_command.args)+" gave return code " + + str(run_command.returncode)) + else: + if capture_output: + return run_command.stdout.decode('utf-8') + +def run_shell_commandc(command, capture_output=True): + """! Run shell command + + Args: + command - list of argument entries (string) + + Returns: + + """ + print("Running "+' '.join(command)) + if any(mark in ' '.join(command) for mark in ['"', "'", '|', '*', '>']): + run_command = subprocess.run( + ' '.join(command), shell=True, capture_output=capture_output + ) + else: + run_command = subprocess.run(command, capture_output=capture_output) + if run_command.returncode != 0: + print("ERROR: "+''.join(run_command.args)+" gave return code " + + str(run_command.returncode)) + else: + if capture_output: + return run_command.stdout.decode('utf-8') + +def format_thresh(thresh): + """! Format threshold with letter and symbol options + + Args: + thresh - the threshold (string) + + Return: + thresh_symbol - threshold with symbols (string) + thresh_letters - treshold with letters (string) + """ + thresh_symbol = ( + thresh.replace('ge', '>=').replace('gt', '>')\ + .replace('eq', '==').replace('ne', '!=')\ + .replace('le', '<=').replace('lt', '<') + ) + thresh_letter = ( + thresh.replace('>=', 'ge').replace('>', 'gt')\ + .replace('==', 'eq').replace('!=', 'ne')\ + .replace('<=', 'le').replace('<', 'lt') + ) + return thresh_symbol, thresh_letter + +def check_file(file_path): + """! Check file exists and not zero size + Args: + file_path - full path to file (string) + Returns: + file_good - full call to METplus (boolean) + """ + if os.path.exists(file_path): + if os.path.getsize(file_path) > 0: + file_good = True + else: + file_good = False + else: + file_good = False + return file_good + +def check_stat_files(job_dict): + """! Check for MET .stat files + + Args: + job_dict - dictionary containing settings + job is running with (strings) + + Returns: + stat_files_exist - if .stat files + exist or not (boolean) + """ + model_stat_file_dir = os.path.join( + job_dict['DATA'], job_dict['VERIF_CASE']+'_'+job_dict['STEP'], + 'METplus_output', job_dict['RUN']+'.'+job_dict['DATE'], + job_dict['MODEL'], job_dict['VERIF_CASE'] + ) + stat_file_list = glob.glob(os.path.join(model_stat_file_dir, '*.stat')) + if len(stat_file_list) != 0: + stat_files_exist = True + else: + stat_files_exist = False + return stat_files_exist + + +def check_pstat_files(job_dict): + """! Check for MET point_stat files + + Args: + job_dict - dictionary containing settings + job is running with (strings) + + Returns: + pstat_files_exist - if point_stat files + exist or not (boolean) + """ + FHR_START = os.environ ['FHR_START'] + FHR_START2= str(FHR_START).zfill(2) + model_stat_file_dir = os.path.join( + job_dict['DATA'], job_dict['VERIF_CASE'], + 'METplus_output', job_dict['VERIF_TYPE'],'point_stat', + job_dict['MODEL']+'.'+job_dict['VDATE'] + ) + + pstat_file = os.path.join( + model_stat_file_dir, 'point_stat_'+job_dict['MODEL']+ + '_'+job_dict['NEST']+'_'+job_dict['VAR_NAME']+ + '_OBS_*0000L_'+job_dict['VDATE']+'_'+job_dict['VHOUR']+'0000V.stat' + ) + + stat_file_list = glob.glob(pstat_file) + + if len(stat_file_list) != 0: + pstat_files_exist = True + else: + pstat_files_exist = False + return pstat_files_exist + + + +def format_filler(unfilled_file_format, valid_time_dt, init_time_dt, + forecast_hour, str_sub_dict): + """! Creates a filled file path from a format + Args: + unfilled_file_format - file naming convention (string) + valid_time_dt - valid time (datetime) + init_time_dt - initialization time (datetime) + forecast_hour - forecast hour (string) + str_sub_dict - other strings to substitue (dictionary) + Returns: + filled_file_format - file_format filled in with verifying + time information (string) + """ + filled_file_format = '/' + format_opt_list = ['lead', 'lead_shift', 'valid', 'valid_shift', + 'init', 'init_shift', 'cycle'] + if len(list(str_sub_dict.keys())) != 0: + format_opt_list = format_opt_list+list(str_sub_dict.keys()) + for filled_file_format_chunk in unfilled_file_format.split('/'): + for format_opt in format_opt_list: + nformat_opt = ( + filled_file_format_chunk.count('{'+format_opt+'?fmt=') + ) + if nformat_opt > 0: + format_opt_count = 1 + while format_opt_count <= nformat_opt: + if format_opt in ['lead_shift', 'valid_shift', + 'init_shift']: + shift = (filled_file_format_chunk \ + .partition('shift=')[2] \ + .partition('}')[0]) + format_opt_count_fmt = ( + filled_file_format_chunk \ + .partition('{'+format_opt+'?fmt=')[2] \ + .rpartition('?')[0] + ) + else: + format_opt_count_fmt = ( + filled_file_format_chunk \ + .partition('{'+format_opt+'?fmt=')[2] \ + .partition('}')[0] + ) + if format_opt == 'valid': + replace_format_opt_count = valid_time_dt.strftime( + format_opt_count_fmt + ) + elif format_opt == 'lead': + if format_opt_count_fmt == '%1H': + if int(forecast_hour) < 10: + replace_format_opt_count = forecast_hour[1] + else: + replace_format_opt_count = forecast_hour + elif format_opt_count_fmt == '%2H': + replace_format_opt_count = forecast_hour.zfill(2) + elif format_opt_count_fmt == '%3H': + replace_format_opt_count = forecast_hour.zfill(3) + else: + replace_format_opt_count = forecast_hour + elif format_opt == 'init': + replace_format_opt_count = init_time_dt.strftime( + format_opt_count_fmt + ) + elif format_opt == 'cycle': + replace_format_opt_count = init_time_dt.strftime( + format_opt_count_fmt + ) + elif format_opt == 'lead_shift': + shift = (filled_file_format_chunk.partition('shift=')[2]\ + .partition('}')[0]) + forecast_hour_shift = str(int(forecast_hour) + + int(shift)) + if format_opt_count_fmt == '%1H': + if int(forecast_hour_shift) < 10: + replace_format_opt_count = ( + forecast_hour_shift[1] + ) + else: + replace_format_opt_count = forecast_hour_shift + elif format_opt_count_fmt == '%2H': + replace_format_opt_count = ( + forecast_hour_shift.zfill(2) + ) + elif format_opt_count_fmt == '%3H': + replace_format_opt_count = ( + forecast_hour_shift.zfill(3) + ) + else: + replace_format_opt_count = forecast_hour_shift + elif format_opt == 'init_shift': + shift = (filled_file_format_chunk.partition('shift=')[2]\ + .partition('}')[0]) + init_shift_time_dt = ( + init_time_dt + td(hours=int(shift)) + ) + replace_format_opt_count = init_shift_time_dt.strftime( + format_opt_count_fmt + ) + elif format_opt == 'valid_shift': + shift = (filled_file_format_chunk.partition('shift=')[2]\ + .partition('}')[0]) + valid_shift_time_dt = ( + valid_time_dt + td(hours=int(shift)) + ) + replace_format_opt_count = valid_shift_time_dt.strftime( + format_opt_count_fmt + ) + else: + replace_format_opt_count = str_sub_dict[format_opt] + if format_opt in ['lead_shift', 'valid_shift', 'init_shift']: + filled_file_format_chunk = ( + filled_file_format_chunk.replace( + '{'+format_opt+'?fmt=' + +format_opt_count_fmt + +'?shift='+shift+'}', + replace_format_opt_count + ) + ) + else: + filled_file_format_chunk = ( + filled_file_format_chunk.replace( + '{'+format_opt+'?fmt=' + +format_opt_count_fmt+'}', + replace_format_opt_count + ) + ) + format_opt_count+=1 + filled_file_format = os.path.join(filled_file_format, + filled_file_format_chunk) + return filled_file_format + +def initalize_job_env_dict(): + """! This initializes a dictionary of environment variables and their + values to be set for the job pulling from environment variables + already set previously + Args: + + Returns: + job_env_dict - dictionary of job settings + """ + os.environ['MET_TMP_DIR'] = os.path.join(os.environ['DATA'], 'tmp') + job_env_var_list = [ + 'machine', 'evs_ver', 'HOMEevs', 'FIXevs', 'USHevs', 'DATA', + 'NET', 'RUN', 'VERIF_CASE', 'STEP', 'COMPONENT', 'evs_run_mode', + 'COMROOT', 'COMIN', 'COMOUT', 'COMOUTsmall', 'COMOUTfinal', 'EVSIN', + 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', + 'MET_ROOT', + 'MET_TMP_DIR', 'MODELNAME', 'JOB_GROUP' + ] + job_env_dict = {} + for env_var in job_env_var_list: + job_env_dict[env_var] = os.environ[env_var] + if env_var in ['LOG_MET_OUTPUT_TO_METPLUS', + ]: + job_env_dict[env_var.lower()] = os.environ[env_var] + return job_env_dict + +def metplus_command(conf_file_name): + """! Write out full call to METplus + Args: + conf_file_name - METplus conf file name (string) + Returns: + metplus_cmd - full call to METplus (string) + """ + run_metplus = os.path.join(os.environ['METPLUS_PATH'], 'ush', + 'run_metplus.py') + machine_conf = os.path.join(os.environ['PARMevs'], 'metplus_config', + 'machine.conf') + conf_file = os.path.join(os.environ['PARMevs'], 'metplus_config', os.environ['STEP'], + os.environ['COMPONENT'], os.environ['VERIF_CASE'], + conf_file_name) + if not os.path.exists(conf_file): + print("ERROR: "+conf_file+" DOES NOT EXIST") + sys.exit(1) + metplus_cmd = run_metplus+' -c '+machine_conf+' -c '+conf_file + return metplus_cmd + +def precip_check_obs_input_output_files(job_dict): + """! Check precip observation input and output files + in COMOUT and DATA + Args: + job_dict - job dictionary + Returns: + all_input_file_exist - if all expected + input files exist + (boolean) + input_files_list - list of input files + (strings) + all_COMOUT_file_exist - if all expected + output COMOUT files + exist (boolean) + COMOUT_files_list - list of output COMOUT + files (strings) + DATA_files_list - list of output DATA + files (strings) + """ + valid_date_dt = datetime.strptime( + job_dict['DATE']+job_dict['valid_hour_start'], + '%Y%m%d%H' + ) + # Expected input file + input_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + if job_dict['job_name'] in ['24hrCCPA', '03hrCCPA', '01hrCCPA']: + nccpa_files = ( + int(job_dict['accum']) + /int(job_dict['ccpa_file_accum']) + ) + n = 1 + while n <= nccpa_files: + nccpa_file = os.path.join( + job_dict['DATA'], 'data', 'ccpa', + f"ccpa.accum{job_dict['ccpa_file_accum'].zfill(2)}hr.v" + +(valid_date_dt + -td(hours=(n-1) + *int(job_dict['ccpa_file_accum'])))\ + .strftime('%Y%m%d%H') + ) + input_files_list.append(nccpa_file) + n+=1 + elif job_dict['JOB_GROUP'] == 'generate_stats': + if job_dict['obs'] == 'ccpa': + input_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + "pcp_combine_ccpa_accum" + +f"{job_dict['accum']}hr_valid" + +f"{valid_date_dt:%Y%m%d%H}.nc") + ) + elif job_dict['obs'] == 'mrms': + input_files_list.append( + os.path.join(job_dict['DATA'], 'data', job_dict['obs'], + f"{job_dict['area']}_MultiSensor_QPE_" + +f"{job_dict['accum']}H_Pass2_00.00_" + +f"{valid_date_dt:%Y%m%d}-" + +f"{valid_date_dt:%H%M%S}.grib2") + ) + input_files_exist_list = [] + for input_file in input_files_list: + if check_file(input_file): + input_files_exist_list.append(True) + else: + input_files_exist_list.append(False) + if all(x == True for x in input_files_exist_list) \ + and len(input_files_exist_list) > 0: + all_input_file_exist = True + else: + all_input_file_exist = False + # Expected output files (in COMOUT and DATA) + COMOUT_files_list = [] + DATA_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + if job_dict['job_name'] in ['24hrCCPA', '03hrCCPA', '01hrCCPA']: + file_name = ("pcp_combine_ccpa_accum" + +f"{job_dict['accum']}hr_valid" + +f"{valid_date_dt:%Y%m%d%H}.nc") + COMOUT_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + DATA_files_list.append( + os.path.join(job_dict['DATA'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + COMOUT_files_exist_list = [] + for COMOUT_file in COMOUT_files_list: + if check_file(COMOUT_file): + COMOUT_files_exist_list.append(True) + else: + COMOUT_files_exist_list.append(False) + if all(x == True for x in COMOUT_files_exist_list) \ + and len(COMOUT_files_exist_list) > 0: + all_COMOUT_file_exist = True + else: + all_COMOUT_file_exist = False + return (all_input_file_exist, input_files_list, \ + all_COMOUT_file_exist, COMOUT_files_list, + DATA_files_list) + + +def precip_check_model_input_output_files(job_dict): + """! Check precip model input and output files + in COMOUT and DATA + Args: + job_dict - job dictionary + Returns: + all_input_file_exist - if all expected + input files exist + (boolean) + input_files_list - list of input files + (strings) + all_COMOUT_file_exist - if all expected + output COMOUT files + exist (boolean) + COMOUT_files_list - list of output COMOUT + files (strings) + DATA_files_list - list of output DATA + files (strings) + """ + valid_date_dt = datetime.strptime( + job_dict['DATE']+job_dict['valid_hour_start'], + '%Y%m%d%H' + ) + init_date_dt = (valid_date_dt + - td(hours=int(job_dict['fcst_hour']))) + # Expected input file + input_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + if job_dict['pcp_combine_method'] == 'SUBTRACT': + for fhr in [job_dict['fcst_hour'], + str(int(job_dict['fcst_hour']) + - int(job_dict['accum']))]: + input_files_list.append( + os.path.join(job_dict['DATA'], 'data', + job_dict['MODELNAME'], + f"{job_dict['MODELNAME']}." + +f"{job_dict['area']}." + +f"init{init_date_dt:%Y%m%d%H}." + +f"f{fhr.zfill(3)}") + ) + elif job_dict['pcp_combine_method'] == 'SUM': + naccum_files = int(job_dict['accum'])/int(job_dict['input_accum']) + n = 1 + while n <= naccum_files: + naccum_file = os.path.join( + job_dict['DATA'], 'data', job_dict['MODELNAME'], + f"{job_dict['MODELNAME']}.{job_dict['area']}." + +f"init{init_date_dt:%Y%m%d%H}.f" + +str(int(job_dict['fcst_hour']) + -((n-1)*int(job_dict['input_accum']))).zfill(3) + ) + input_files_list.append(naccum_file) + n+=1 + elif job_dict['JOB_GROUP'] == 'generate_stats': + input_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + f"pcp_combine_{job_dict['MODELNAME']}_" + +f"accum{job_dict['accum']}hr_" + +f"{job_dict['area']}_" + +f"init{init_date_dt:%Y%m%d%H}_" + +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") + ) + input_files_exist_list = [] + for input_file in input_files_list: + if check_file(input_file): + input_files_exist_list.append(True) + else: + input_files_exist_list.append(False) + if all(x == True for x in input_files_exist_list) \ + and len(input_files_exist_list) > 0: + all_input_file_exist = True + else: + all_input_file_exist = False + # Expected output files (in COMOUT and DATA) + COMOUT_files_list = [] + DATA_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + file_name = (f"pcp_combine_{job_dict['MODELNAME']}_" + +f"accum{job_dict['accum']}hr_" + +f"{job_dict['area']}_" + +f"init{init_date_dt:%Y%m%d%H}_" + +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") + COMOUT_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + DATA_files_list.append( + os.path.join(job_dict['DATA'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + elif job_dict['JOB_GROUP'] == 'generate_stats': + file_name = (f"grid_stat_{job_dict['job_name']}_" + f"{job_dict['fcst_hour'].zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H%M%S}V.stat") + COMOUT_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + DATA_files_list.append( + os.path.join(job_dict['DATA'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + COMOUT_files_exist_list = [] + for COMOUT_file in COMOUT_files_list: + if check_file(COMOUT_file): + COMOUT_files_exist_list.append(True) + else: + COMOUT_files_exist_list.append(False) + if all(x == True for x in COMOUT_files_exist_list) \ + and len(COMOUT_files_exist_list) > 0: + all_COMOUT_file_exist = True + else: + all_COMOUT_file_exist = False + return (all_input_file_exist, input_files_list, \ + all_COMOUT_file_exist, COMOUT_files_list, + DATA_files_list) + +def snowfall_check_obs_input_output_files(job_dict): + """! Check snowfall observation input and output files + in COMOUT and DATA + Args: + job_dict - job dictionary + Returns: + all_input_file_exist - if all expected + input files exist + (boolean) + input_files_list - list of input files + (strings) + all_COMOUT_file_exist - if all expected + output COMOUT files + exist (boolean) + COMOUT_files_list - list of output COMOUT + files (strings) + DATA_files_list - list of output DATA + files (strings) + """ + valid_date_dt = datetime.strptime( + job_dict['DATE']+job_dict['valid_hour_start'], + '%Y%m%d%H' + ) + # Expected input file + input_files_list = [] + if job_dict['JOB_GROUP'] == 'generate_stats': + if job_dict['obs'] == 'nohrsc': + input_files_list.append( + os.path.join(job_dict['DATA'], 'data', 'nohrsc', + f"nohrsc.accum{job_dict['accum']}hr." + +f"v{valid_date_dt:%Y%m%d%H}") + ) + input_files_exist_list = [] + for input_file in input_files_list: + if check_file(input_file): + input_files_exist_list.append(True) + else: + input_files_exist_list.append(False) + if all(x == True for x in input_files_exist_list) \ + and len(input_files_exist_list) > 0: + all_input_file_exist = True + else: + all_input_file_exist = False + # Expected output files (in COMOUT and DATA) + COMOUT_files_list = [] + DATA_files_list = [] + # + COMOUT_files_exist_list = [] + for COMOUT_file in COMOUT_files_list: + if check_file(COMOUT_file): + COMOUT_files_exist_list.append(True) + else: + COMOUT_files_exist_list.append(False) + if all(x == True for x in COMOUT_files_exist_list) \ + and len(COMOUT_files_exist_list) > 0: + all_COMOUT_file_exist = True + else: + all_COMOUT_file_exist = False + return (all_input_file_exist, input_files_list, \ + all_COMOUT_file_exist, COMOUT_files_list, + DATA_files_list) + +def snowfall_check_model_input_output_files(job_dict): + """! Check snowfall model input and output files + in COMOUT and DATA + Args: + job_dict - job dictionary + Returns: + all_input_file_exist - if all expected + input files exist + (boolean) + input_files_list - list of input files + (strings) + all_COMOUT_file_exist - if all expected + output COMOUT files + exist (boolean) + COMOUT_files_list - list of output COMOUT + files (strings) + DATA_files_list - list of output DATA + files (strings) + """ + valid_date_dt = datetime.strptime( + job_dict['DATE']+job_dict['valid_hour_start'], + '%Y%m%d%H' + ) + init_date_dt = (valid_date_dt + - td(hours=int(job_dict['fcst_hour']))) + # Expected input file + input_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + if job_dict['pcp_combine_method'] in ['SUBTRACT', 'USER_DEFINED']: + for fhr in [job_dict['fcst_hour'], + str(int(job_dict['fcst_hour']) + - int(job_dict['accum']))]: + input_files_list.append( + os.path.join(job_dict['DATA'], 'data', + job_dict['MODELNAME'], + f"{job_dict['MODELNAME']}." + +f"init{init_date_dt:%Y%m%d%H}." + +f"f{fhr.zfill(3)}") + ) + elif job_dict['JOB_GROUP'] == 'generate_stats': + input_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + f"pcp_combine_{job_dict['MODELNAME']}_" + +f"accum{job_dict['accum']}hr_" + +f"{job_dict['snow_var']}_" + +f"init{init_date_dt:%Y%m%d%H}_" + +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") + ) + input_files_exist_list = [] + for input_file in input_files_list: + if check_file(input_file): + input_files_exist_list.append(True) + else: + input_files_exist_list.append(False) + if all(x == True for x in input_files_exist_list) \ + and len(input_files_exist_list) > 0: + all_input_file_exist = True + else: + all_input_file_exist = False + # Expected output files (in COMOUT and DATA) + COMOUT_files_list = [] + DATA_files_list = [] + if job_dict['JOB_GROUP'] == 'assemble_data': + file_name = (f"pcp_combine_{job_dict['MODELNAME']}_" + +f"accum{job_dict['accum']}hr_" + +f"{job_dict['snow_var']}_" + +f"init{init_date_dt:%Y%m%d%H}_" + +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") + COMOUT_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + DATA_files_list.append( + os.path.join(job_dict['DATA'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + elif job_dict['JOB_GROUP'] == 'generate_stats': + file_name = (f"grid_stat_{job_dict['job_name']}_" + f"{job_dict['fcst_hour'].zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H%M%S}V.stat") + COMOUT_files_list.append( + os.path.join(job_dict['COMOUT'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + DATA_files_list.append( + os.path.join(job_dict['DATA'], + f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", + job_dict['MODELNAME'], job_dict['VERIF_CASE'], + file_name) + ) + COMOUT_files_exist_list = [] + for COMOUT_file in COMOUT_files_list: + if check_file(COMOUT_file): + COMOUT_files_exist_list.append(True) + else: + COMOUT_files_exist_list.append(False) + if all(x == True for x in COMOUT_files_exist_list) \ + and len(COMOUT_files_exist_list) > 0: + all_COMOUT_file_exist = True + else: + all_COMOUT_file_exist = False + return (all_input_file_exist, input_files_list, \ + all_COMOUT_file_exist, COMOUT_files_list, + DATA_files_list) + + +def get_completed_jobs(completed_jobs_file): + completed_jobs = set() + if os.path.exists(completed_jobs_file): + with open(completed_jobs_file, 'r') as f: + completed_jobs = set(f.read().splitlines()) + return completed_jobs + +def mark_job_completed(completed_jobs_file, job_name): + with open(completed_jobs_file, 'a') as f: + f.write(job_name + "\n") + +# Construct a file name given a template +def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", + VDATE="YYYYmmdd", VHOUR="HH", VDATEHOUR="YYYYmmddHH", + VDATEm1H="YYYYmmdd", VDATEHOURm1H="YYYYmmddHH", + FHR="HH", LVL="0", OFFSET="HH"): + template_str = template_str.replace('{IDATE}', IDATE) + template_str = template_str.replace('{IHOUR}', IHOUR) + template_str = template_str.replace('{VDATE}', VDATE) + template_str = template_str.replace('{VHOUR}', VHOUR) + template_str = template_str.replace('{VDATEHOUR}', VDATEHOUR) + template_str = template_str.replace('{VDATEm1H}', VDATEm1H) + template_str = template_str.replace('{VDATEHOURm1H}', VDATEHOURm1H) + template_str = template_str.replace('{FHR}', FHR) + template_str = template_str.replace('{LVL}', LVL) + template_str = template_str.replace('{OFFSET}', OFFSET) + return template_str + +# Create a list of prepbufr file paths +def get_prepbufr_templates(indir, vdates, paths=[], obsname='both', already_preprocessed=False): + ''' + indir - (str) Input directory for prepbufr file data + vdates - (datetime object) List of datetimes used to fill templates + paths - (list of str) list of paths to append the prepbufr paths to + Default is empty. + ''' + prepbufr_templates = [] + prepbufr_paths = [] + for v, vdate in enumerate(vdates): + vh = vdate.strftime('%H') + vd = vdate.strftime('%Y%m%d') + if vh in ['00', '03', '06', '09', '12', '15', '18', '21']: + if vh in ['03', '09', '15', '21']: + offsets = ['03'] + elif vh in ['00', '06', '12', '18']: + offsets = ['00', '06'] + if obsname in ['both', 'raob']: + if not already_preprocessed: + prepbufr_templates.append(os.path.join( + indir, + 'gdas.{VDATE}', + '{VHOUR}', + 'atmos', + 'gdas.t{VHOUR}z.prepbufr' + )) + else: + prepbufr_templates.append(os.path.join( + indir, + 'gdas.t{VHOUR}z.prepbufr' + )) + for offset in offsets: + use_vdate = vdate + td(hours=int(offset)) + use_vd = use_vdate.strftime('%Y%m%d') + use_vh = use_vdate.strftime('%H') + if obsname in ['both', 'metar']: + if not already_preprocessed: + template = os.path.join( + indir, + 'nam.{VDATE}', + 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' + ) + else: + template = os.path.join( + indir, + 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' + ) + prepbufr_paths.append(fname_constructor( + template, VDATE=use_vd, VHOUR=use_vh, OFFSET=offset + )) + for template in prepbufr_templates: + prepbufr_paths.append(fname_constructor( + template, VDATE=vd, VHOUR=vh + )) + return np.concatenate((paths, np.unique(prepbufr_paths))) + +def preprocess_prepbufr(indir, fname, workdir, outdir, subsets): + if os.path.exists(os.path.join(outdir, fname)): + print(f"{fname} exists in {outdir} so we can skip preprocessing.") + else: + wd = os.getcwd() + os.chdir(workdir) + if os.path.isfile(os.path.join(indir, fname)): + run_shell_command( + [ + os.path.join(os.environ['bufr_ROOT'], 'bin', 'split_by_subset'), + os.path.join(indir, fname) + ] + ) + if all([os.path.isfile(subset) for subset in subsets]): + run_shell_command( + np.concatenate(( + ['cat'], subsets, ['>>', os.path.join(outdir, fname)] + )) + ) + else: + raise FileNotFoundError( + f"The following prepbufr subsets do not exist in {workdir}: " + + ', '.join([subset for subset in subsets if not os.path.isfile(subset)]) + + ". Cannot concatenate subsets." + ) + else: + print( + "WARNING: The following file does not exist: " + + f"{os.path.join(indir, fname)}." + + " Skipping split by subset." + ) + os.chdir(wd) + From 19224b550fc6bece3c55028f54a1f4170f8ed40a Mon Sep 17 00:00:00 2001 From: perry shafran Date: Thu, 9 Jan 2025 21:40:42 +0000 Subject: [PATCH 4/8] Adding in new stuff for restart --- ush/mesoscale/mesoscale_create_output_dirs.py | 2 +- ...oscale_stats_grid2obs_create_job_script.py | 29 ++++++++++++++++++- ush/mesoscale/mesoscale_util.py | 24 +++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/ush/mesoscale/mesoscale_create_output_dirs.py b/ush/mesoscale/mesoscale_create_output_dirs.py index 1d83935f7..2e72bf7d5 100644 --- a/ush/mesoscale/mesoscale_create_output_dirs.py +++ b/ush/mesoscale/mesoscale_create_output_dirs.py @@ -246,7 +246,7 @@ DATA, VERIF_CASE, 'METplus_output', 'gather_small' ) COMOUT_restart_base_dir = os.path.join( - RESTART_DIR, 'METplus_output' + RESTART_DIR, 'METplus_output', 'gather_small' ) if job_type == 'gather2': working_output_base_dir = os.path.join( diff --git a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py index 273d3dee0..c136505e0 100644 --- a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py +++ b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py @@ -39,7 +39,6 @@ VDATE = os.environ['VDATE'] MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] -MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] metplus_launcher = 'run_metplus.py' machine_conf = os.path.join( @@ -62,6 +61,9 @@ MIN_IHOUR = os.environ['MIN_IHOUR'] COMINobs = os.environ['COMINobs'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) USHevs = os.environ['USHevs'] SKIP_IF_OUTPUT_EXISTS = os.environ['SKIP_IF_OUTPUT_EXISTS'] if NEST == 'spc_otlk': @@ -86,6 +88,9 @@ if NEST not in ['firewx', 'spc_otlk']: MASK_POLY_LIST = os.environ['MASK_POLY_LIST'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) GRID = os.environ['GRID'] USHevs = os.environ['USHevs'] if NEST == 'spc_otlk': @@ -93,9 +98,15 @@ elif job_type == 'gather': VERIF_TYPE = os.environ['VERIF_TYPE'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type in ['gather2','gather3']: VERIF_TYPE = os.environ['VERIF_TYPE'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) # Get expanded details from variable name if job_type == 'generate': @@ -371,6 +382,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -389,6 +401,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -419,6 +432,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -442,6 +456,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -477,6 +492,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -499,6 +515,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -522,6 +539,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -545,6 +563,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -567,6 +586,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -590,6 +610,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -628,6 +649,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -652,6 +674,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'vx_mask=\\\"${NEST}\\\", ' @@ -686,6 +709,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'met_tool=\\\"stat_analysis\\\", ' @@ -708,6 +732,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'verif_type=\\\"${VERIF_TYPE}\\\", ' + 'met_tool=\\\"stat_analysis\\\", ' @@ -740,6 +765,7 @@ f'#python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'met_tool=\\\"stat_analysis\\\", ' + 'vdate=\\\"${VDATE}\\\", ' @@ -762,6 +788,7 @@ f'python -c ' + '\"import mesoscale_util as cutil; cutil.copy_data_to_restart(' + '\\\"${DATA}\\\", \\\"${RESTART_DIR}\\\", ' + + f'njob=\\\"{njob}\\\", ' + 'verif_case=\\\"${VERIF_CASE}\\\", ' + 'met_tool=\\\"stat_analysis\\\", ' + 'vdate=\\\"${VDATE}\\\", ' diff --git a/ush/mesoscale/mesoscale_util.py b/ush/mesoscale/mesoscale_util.py index c7acebc0b..b67b65f60 100644 --- a/ush/mesoscale/mesoscale_util.py +++ b/ush/mesoscale/mesoscale_util.py @@ -405,6 +405,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'reformat', + f'job{njob}', verif_type, vx_mask, met_tool, @@ -421,6 +424,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'reformat', + f'job{njob}', verif_type, met_tool, f'{vx_mask}.{vdate}', @@ -503,6 +509,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'reformat', + f'job{njob}', verif_type, vx_mask, met_tool, @@ -547,6 +556,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, ihour = idt.strftime('%H') sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'reformat', + f'job{njob}', verif_type, met_tool, )) @@ -564,6 +576,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'generate', + f'job{njob}', verif_type, met_tool, f'{model}.{vdate}' @@ -584,6 +599,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'generate' + f'job{njob}', verif_type, met_tool, f'{model}.{vdate}' @@ -605,6 +623,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'gather', + f'job{njob}', 'gather_small', met_tool, f'{model}.{vdate}' @@ -624,6 +645,9 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, raise TypeError(e) sub_dirs.append(os.path.join( 'METplus_output', + 'workdirs', + 'gather2', + f'job{njob}', met_tool, f'{model}.{vdate}' )) From 3c3b145de1f373cabd53bb158905bb2c267bbf16 Mon Sep 17 00:00:00 2001 From: perry shafran Date: Tue, 14 Jan 2025 18:45:32 +0000 Subject: [PATCH 5/8] More changes to correct the restart with mpmd --- .../mesoscale_create_output_dirs_orig.py | 496 --------- ush/mesoscale/mesoscale_util.py | 78 +- ush/mesoscale/mesoscale_util_save.py | 949 ------------------ 3 files changed, 65 insertions(+), 1458 deletions(-) delete mode 100644 ush/mesoscale/mesoscale_create_output_dirs_orig.py delete mode 100644 ush/mesoscale/mesoscale_util_save.py diff --git a/ush/mesoscale/mesoscale_create_output_dirs_orig.py b/ush/mesoscale/mesoscale_create_output_dirs_orig.py deleted file mode 100644 index 393470bfe..000000000 --- a/ush/mesoscale/mesoscale_create_output_dirs_orig.py +++ /dev/null @@ -1,496 +0,0 @@ -#!/usr/bin/env python3 -# ============================================================================= -# -# NAME: mesoscale_create_output_dirs.py -# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# Roshan Shrestha, roshan.shrestha@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# PURPOSE: Define working/ output directories and create them if they don't -# exist. -# DEPENDENCIES: os.path.join([ -# SCRIPTSevs,COMPONENT,STEP, -# "_".join(["exevs",MODELNAME,VERIF_CASE,STEP+".sh"] -# )] -# -# ============================================================================= - -import os -import re -from datetime import datetime, timedelta as td -from mesoscale_plots_grid2obs_graphx_defs import graphics as graphics_g2o -from mesoscale_plots_precip_graphx_defs import graphics as graphics_pcp -from mesoscale_plots_headline_graphx_defs import graphics as graphics_hdl -from mesoscale_plots_snowfall_graphx_defs import graphics as graphics_sno -import mesoscale_util as cutil - -print(f"BEGIN: {os.path.basename(__file__)}") - -# Read in environment variables -evs_ver = os.environ['evs_ver'] -EVSIN = os.environ['EVSIN'] -COMOUT = os.environ['COMOUT'] -DATA = os.environ['DATA'] -NET = os.environ['NET'] -RUN = os.environ['RUN'] -COMPONENT = os.environ['COMPONENT'] -VERIF_CASE = os.environ['VERIF_CASE'] -STEP = os.environ['STEP'] -MODELNAME = os.environ['MODELNAME'] -VDATE = os.environ['VDATE'] -vdate_dt = datetime.strptime(VDATE, '%Y%m%d') -if VERIF_CASE == "precip": - if STEP == 'prep': - FHR_END_FULL = os.environ['FHR_END_FULL'] - FHR_END_SHORT = os.environ['FHR_END_SHORT'] - fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) - start_date_dt = vdate_dt - td(hours=fhr_end_max) - VERIF_TYPE = os.environ['VERIF_TYPE'] - OBSNAME = os.environ['OBSNAME'] - elif STEP == 'stats': - FHR_END_FULL = os.environ['FHR_END_FULL'] - FHR_END_SHORT = os.environ['FHR_END_SHORT'] - fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) - start_date_dt = vdate_dt - td(hours=fhr_end_max) - VERIF_TYPE = os.environ['VERIF_TYPE'] - OBSNAME = os.environ['OBSNAME'] - COMOUTsmall = os.environ['COMOUTsmall'] - elif STEP == 'plots': - all_eval_periods = cutil.get_all_eval_periods(graphics_pcp) - COMOUTplots = os.environ['COMOUTplots'] -elif VERIF_CASE == "grid2obs": - if STEP == 'prep': - NEST = os.environ['NEST'] - if STEP == 'stats': - NEST = os.environ['NEST'] - FHR_END_FULL = os.environ['FHR_END_FULL'] - FHR_END_SHORT = os.environ['FHR_END_SHORT'] - fhr_end_max = max(int(FHR_END_FULL), int(FHR_END_SHORT)) - start_date_dt = vdate_dt - td(hours=fhr_end_max) - VERIF_TYPE = os.environ['VERIF_TYPE'] - OBSNAME = os.environ['OBSNAME'] - COMOUTsmall = os.environ['COMOUTsmall'] - elif STEP == 'plots': - all_eval_periods = cutil.get_all_eval_periods(graphics_g2o) - COMOUTplots = os.environ['COMOUTplots'] -elif VERIF_CASE == "headline": - if STEP == 'plots': - all_eval_periods = cutil.get_all_eval_periods(graphics_hdl) - COMOUTplots = os.environ['COMOUTplots'] -elif VERIF_CASE == "snowfall": - if STEP == 'prep': - pass - elif STEP == 'stats': - pass - elif STEP == 'plots': - all_eval_periods = cutil.get_all_eval_periods(graphics_sno) - COMOUTplots = os.environ['COMOUTplots'] -if STEP == 'stats': - job_type = os.environ['job_type'] -if STEP == 'plots': - RESTART_DIR = os.environ['RESTART_DIR'] - - -# Define data base directorie -data_base_dir = os.path.join(DATA, VERIF_CASE, 'data') -data_dir_list = [data_base_dir] -if VERIF_CASE == 'precip': - if STEP == 'prep': - data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) - data_dir_list.append(os.path.join(data_base_dir, OBSNAME)) - if STEP == 'stats': - data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) - data_dir_list.append(os.path.join(data_base_dir, OBSNAME)) -elif VERIF_CASE == 'grid2obs': - if STEP == 'stats': - data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) - data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'merged_ptype')) - data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'tmp')) - data_dir_list.append(os.path.join( - data_base_dir, OBSNAME, 'prepbufr' - )) -elif VERIF_CASE == 'snowfall': - if STEP == 'stats': - pass - -# Create data directories and subdirectories -for data_dir in data_dir_list: - if not os.path.exists(data_dir): - print(f"Creating data directory: {data_dir}") - os.makedirs(data_dir, mode=0o755) - -# Create job script base directory -job_scripts_dirs = [] -if STEP == 'prep': - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, STEP, 'prep_job_scripts')) -if STEP == 'stats': - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'reformat')) - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'generate')) - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather')) - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather2')) - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, 'METplus_job_scripts', 'gather3')) -if STEP == 'plots': - job_scripts_dirs.append(os.path.join(DATA, VERIF_CASE, STEP, 'plotting_job_scripts')) -for job_scripts_dir in job_scripts_dirs: - if not os.path.exists(job_scripts_dir): - print(f"Creating job script directory: {job_scripts_dir}") - os.makedirs(job_scripts_dir, mode=0o755) - -# Define working and COMOUT directories -working_dir_list = [] -COMOUT_dir_list = [] -if STEP == 'prep': - if VERIF_CASE == 'grid2obs': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join( - working_output_base_dir, 'data', - NEST+'.'+vdate_dt.strftime('%Y%m%d') - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - NEST+'.'+vdate_dt.strftime('%Y%m%d') - )) -elif STEP == 'stats': - if VERIF_CASE == 'precip': - if job_type == 'reformat': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE - ) - if job_type == 'generate': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE - ) - if job_type == 'gather': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', 'gather_small' - ) - if job_type == 'gather2': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output' - ) - if job_type == 'gather3': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output' - ) - working_dir_list.append(working_output_base_dir) - if job_type == 'reformat': - working_dir_list.append(os.path.join( - working_output_base_dir, 'pcp_combine', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'pcp_combine', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'pcp_combine', 'tmp' - )) - if job_type == 'generate': - working_dir_list.append(os.path.join( - working_output_base_dir, 'grid_stat', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'grid_stat', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'grid_stat', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'grid_stat', - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - if job_type in ['gather', 'gather2', 'gather3']: - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - if job_type == 'reformat': - working_dir_list.append(os.path.join( - working_output_base_dir, 'pcp_combine', - OBSNAME+'.'+date_dt.strftime('%Y%m%d') - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'pcp_combine', - MODELNAME+'.'+date_dt.strftime('init%Y%m%d') - )) - elif VERIF_CASE == "grid2obs": - if job_type == 'reformat': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE - ) - if job_type == 'generate': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE - ) - if job_type == 'gather': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', 'gather_small' - ) - if job_type == 'gather2': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output' - ) - if job_type == 'gather3': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output' - ) - working_dir_list.append(working_output_base_dir) - if job_type == 'reformat': - working_dir_list.append(os.path.join( - working_output_base_dir, NEST, 'pb2nc', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, NEST, 'pb2nc', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, NEST, 'pb2nc', 'tmp' - )) - if NEST in ['spc_otlk', 'firewx']: - working_dir_list.append(os.path.join( - working_output_base_dir, 'genvxmask', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'genvxmask', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'genvxmask', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'genvxmask', - NEST+'.'+vdate_dt.strftime('%Y%m%d') - )) - if job_type == 'generate': - working_dir_list.append(os.path.join( - working_output_base_dir, 'regrid_data_plane', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'regrid_data_plane', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'regrid_data_plane', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'regrid_data_plane', - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'point_stat', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'point_stat', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'point_stat', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'point_stat', - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - if job_type in ['gather', 'gather2', 'gather3']: - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'confs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'logs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', 'tmp' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'stat_analysis', - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - COMOUT_dir_list.append(os.path.join( - COMOUTsmall, - 'gather_small' - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - if job_type == 'reformat': - working_dir_list.append(os.path.join( - working_output_base_dir, NEST, 'pb2nc', - OBSNAME+'.'+vdate_dt.strftime('%Y%m%d') - )) - working_dir_list.append(os.path.join( - working_output_base_dir, NEST, 'pb2nc', - MODELNAME+'.'+vdate_dt.strftime('init%Y%m%d') - )) - elif VERIF_CASE == "snowfall": - pass - working_dir_list.append(os.path.join( - DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type - )) -elif STEP == 'plots': - if VERIF_CASE == 'grid2obs': - - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join( - working_output_base_dir, 'data' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'workdirs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'logs' - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, VERIF_CASE - )) - for plot_group in ['cape', 'ceil_vis','sfc_upper']: - for eval_period in all_eval_periods: - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', str(plot_group).lower(), - str(eval_period).lower() - )) - COMOUT_dir_list.append(os.path.join( - RESTART_DIR, str(plot_group).lower(), - str(eval_period).lower() - )) - if VERIF_CASE == 'precip': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join( - working_output_base_dir, 'data' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'workdirs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'logs' - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, VERIF_CASE - )) - for plot_group in ['precip']: - for eval_period in all_eval_periods: - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', str(plot_group).lower(), - str(eval_period).lower() - )) - if not str(eval_period).lower() == 'na' : - COMOUT_dir_list.append(os.path.join( - RESTART_DIR, str(plot_group).lower(), - str(eval_period).lower() - )) - elif VERIF_CASE == 'snowfall': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join( - working_output_base_dir, 'data' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'workdirs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'logs' - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, VERIF_CASE - )) - for plot_group in ['precip']: - for eval_period in all_eval_periods: - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', str(plot_group).lower(), - str(eval_period).lower() - )) - COMOUT_dir_list.append(os.path.join( - RESTART_DIR, str(plot_group).lower(), - str(eval_period).lower() - )) - elif VERIF_CASE == 'headline': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join( - working_output_base_dir, 'data' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'workdirs' - )) - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', 'logs' - )) - COMOUT_dir_list.append(os.path.join( - COMOUT, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, - )) - COMOUT_dir_list.append(os.path.join( - COMOUTplots, VERIF_CASE - )) - for plot_group in ['sfc_upper']: - for eval_period in all_eval_periods: - working_dir_list.append(os.path.join( - working_output_base_dir, 'out', str(plot_group).lower(), - str(eval_period).lower() - )) - COMOUT_dir_list.append(os.path.join( - RESTART_DIR, str(plot_group).lower(), - str(eval_period).lower() - )) -# Create working output and COMOUT directories -for working_dir in working_dir_list: - if not os.path.exists(working_dir): - print(f"Creating working output directory: {working_dir}") - os.makedirs(working_dir, mode=0o755, exist_ok=True) - else: - print(f"Tried creating working output directory but already exists: {working_dir}") -for COMOUT_dir in COMOUT_dir_list: - if not os.path.exists(COMOUT_dir): - print(f"Creating COMOUT directory: {COMOUT_dir}") - os.makedirs(COMOUT_dir, mode=0o755, exist_ok=True) - -print(f"END: {os.path.basename(__file__)}") diff --git a/ush/mesoscale/mesoscale_util.py b/ush/mesoscale/mesoscale_util.py index b67b65f60..ff0af39e9 100644 --- a/ush/mesoscale/mesoscale_util.py +++ b/ush/mesoscale/mesoscale_util.py @@ -392,7 +392,8 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, job_type=None, var_name=None, vhour=None, fhr_start=None, fhr_end=None, fhr_incr=None, njob=None, acc=None, nbrhd=None): - sub_dirs = [] + sub_dirs_in = [] + sub_dirs_out = [] copy_files = [] if met_tool == "ascii2nc": check_if_none = [ @@ -403,7 +404,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'reformat', @@ -412,6 +413,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, vx_mask, met_tool, )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + vx_mask, + met_tool, + )) copy_files.append(f'{verif_type}.{vdate}{vhour}.nc') elif met_tool == 'genvxmask': check_if_none = [ @@ -422,7 +429,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'reformat', @@ -431,6 +438,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{vx_mask}.{vdate}', )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{vx_mask}.{vdate}', + )) for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): copy_files.append(f'{vx_mask}_t{vhour}z_f{str(fhr).zfill(2)}.nc') elif met_tool == 'grid_stat': @@ -484,7 +497,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( + 'data', + model, + met_tool, + )) + sub_dirs_out.append(os.path.join( 'data', model, met_tool, @@ -507,7 +525,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'reformat', @@ -516,6 +534,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, vx_mask, met_tool, )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + vx_mask, + met_tool, + )) copy_files.append(f'prepbufr.*.{vdate}{vhour}.nc') elif met_tool == 'pcp_combine': if verif_case == "snowfall": @@ -554,7 +578,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, idt = vdt - td(hours=int(fhr)) idate = idt.strftime('%Y%m%d') ihour = idt.strftime('%H') - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'reformat', @@ -562,6 +586,11 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, verif_type, met_tool, )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + )) copy_files.append( f'{model}.init{idate}.t{ihour}z.f{str(fhr).zfill(3)}.a{acc}h.{vx_mask}.nc' ) @@ -574,7 +603,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'generate', @@ -583,6 +612,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_{vx_mask}_{var_name}_OBS*_{str(fhr).zfill(2)}0000L_{vdate}_' @@ -597,7 +632,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'generate' @@ -606,6 +641,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + verif_type, + met_tool, + f'{model}.{vdate}' + )) for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_t{vhour}z_{verif_type}_{vx_mask}_job{njob}_' @@ -621,7 +662,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'gather', @@ -630,6 +671,12 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + 'gather_small', + met_tool, + f'{model}.{vdate}' + )) copy_files.append( f'{net}.{step}.{model}.{run}.{verif_case}.{verif_type}' + f'.v{vdate}.stat' @@ -643,7 +690,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - sub_dirs.append(os.path.join( + sub_dirs_in.append(os.path.join( 'METplus_output', 'workdirs', 'gather2', @@ -651,15 +698,20 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) + sub_dirs_out.append(os.path.join( + 'METplus_output', + met_tool, + f'{model}.{vdate}' + )) copy_files.append( f'{net}.{step}.{model}.{run}.{verif_case}.v{vdate}.c{vhr}z.stat' ) - for sub_dir in sub_dirs: + for s, sub_dir_out in enumerate(sub_dirs_out): for copy_file in copy_files: origin_path = os.path.join( - data_dir, verif_case, sub_dir, copy_file + data_dir, verif_case, sub_dirs_in[s], copy_file ) - dest_path = os.path.join(restart_dir, sub_dir) + dest_path = os.path.join(restart_dir, sub_dir_out) if not glob.glob(origin_path): continue if not os.path.exists(dest_path): diff --git a/ush/mesoscale/mesoscale_util_save.py b/ush/mesoscale/mesoscale_util_save.py deleted file mode 100644 index 570eee8e4..000000000 --- a/ush/mesoscale/mesoscale_util_save.py +++ /dev/null @@ -1,949 +0,0 @@ -#!/usr/bin/env python3 -# ============================================================================= -# -# NAME: mesoscale_util.py -# CONTRIBUTOR(S): RS, roshan.shrestha@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# Mallory Row, mallory.row@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# PURPOSE: Various Utilities for EVS MESOSCALE Verification -# -# ============================================================================= - -import os -import sys -from datetime import datetime, timedelta as td -import numpy as np -import glob -import subprocess -from collections.abc import Iterable - -def flatten(xs): - for x in xs: - if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): - yield from flatten(x) - else: - yield x - -def get_data_type(fname): - data_type_dict = { - 'PrepBUFR': { - 'and':[''], - 'or':['prepbufr'], - 'not':[], - 'type': 'anl' - }, - 'SPC Outlook Area': { - 'and':[''], - 'or':['spc_otlk'], - 'not':[], - 'type': 'gen' - }, - 'NAM': { - 'and':[''], - 'or':['nam'], - 'not':[], - 'type': 'fcst' - }, - 'RAP': { - 'and':[''], - 'or':['rap'], - 'not':[], - 'type': 'fcst' - }, - } - for k in data_type_dict: - if not data_type_dict[k]['and'] or not any(data_type_dict[k]['and']): - data_type_dict[k]['and'] = [''] - if not data_type_dict[k]['or'] or not any(data_type_dict[k]['or']): - data_type_dict[k]['or'] = [''] - if not data_type_dict[k]['not'] or not any(data_type_dict[k]['not']): - data_type_dict[k]['not'] = [] - data_names = [ - k for k in data_type_dict - if ( - all(map(fname.__contains__, data_type_dict[k]['and'])) - and any(map(fname.__contains__, data_type_dict[k]['or'])) - and not any(map(fname.__contains__, data_type_dict[k]['not'])) - ) - ] - if len(data_names) == 1: - data_name = data_names[0] - return data_name, data_type_dict[data_name]['type'] - else: - data_name = "Unknown" - return data_name, 'unk' - -def get_all_eval_periods(graphics): - all_eval_periods = [] - for component in graphics: - for verif_case in graphics[component]: - for verif_type in graphics[component][verif_case]: - verif_type_dict = graphics[component][verif_case][verif_type] - for models in verif_type_dict: - for plot_type in verif_type_dict[models]: - all_eval_periods.append( - verif_type_dict[models][plot_type]['EVAL_PERIODS'] - ) - return np.unique(np.hstack(all_eval_periods)) - -def get_fhr_start(vhour, acc, fhr_incr, min_ihour): - fhr_start = ( - float(vhour) + float(min_ihour) - + ( - float(fhr_incr) - * np.ceil( - (float(acc)-float(vhour)-float(min_ihour)) - / float(fhr_incr) - ) - ) - ) - return int(fhr_start) - -def run_shell_command(command, capture_output=False): - """! Run shell command - - Args: - command - list of argument entries (string) - - Returns: - - """ - print("Running "+' '.join(command)) - if any(mark in ' '.join(command) for mark in ['"', "'", '|', '*', '>']): - run_command = subprocess.run( - ' '.join(command), shell=True, capture_output=capture_output - ) - else: - run_command = subprocess.run(command, capture_output=capture_output) - if run_command.returncode != 0: - print("ERROR: "+''.join(run_command.args)+" gave return code " - + str(run_command.returncode)) - else: - if capture_output: - return run_command.stdout.decode('utf-8') - -def run_shell_commandc(command, capture_output=True): - """! Run shell command - - Args: - command - list of argument entries (string) - - Returns: - - """ - print("Running "+' '.join(command)) - if any(mark in ' '.join(command) for mark in ['"', "'", '|', '*', '>']): - run_command = subprocess.run( - ' '.join(command), shell=True, capture_output=capture_output - ) - else: - run_command = subprocess.run(command, capture_output=capture_output) - if run_command.returncode != 0: - print("ERROR: "+''.join(run_command.args)+" gave return code " - + str(run_command.returncode)) - else: - if capture_output: - return run_command.stdout.decode('utf-8') - -def format_thresh(thresh): - """! Format threshold with letter and symbol options - - Args: - thresh - the threshold (string) - - Return: - thresh_symbol - threshold with symbols (string) - thresh_letters - treshold with letters (string) - """ - thresh_symbol = ( - thresh.replace('ge', '>=').replace('gt', '>')\ - .replace('eq', '==').replace('ne', '!=')\ - .replace('le', '<=').replace('lt', '<') - ) - thresh_letter = ( - thresh.replace('>=', 'ge').replace('>', 'gt')\ - .replace('==', 'eq').replace('!=', 'ne')\ - .replace('<=', 'le').replace('<', 'lt') - ) - return thresh_symbol, thresh_letter - -def check_file(file_path): - """! Check file exists and not zero size - Args: - file_path - full path to file (string) - Returns: - file_good - full call to METplus (boolean) - """ - if os.path.exists(file_path): - if os.path.getsize(file_path) > 0: - file_good = True - else: - file_good = False - else: - file_good = False - return file_good - -def check_stat_files(job_dict): - """! Check for MET .stat files - - Args: - job_dict - dictionary containing settings - job is running with (strings) - - Returns: - stat_files_exist - if .stat files - exist or not (boolean) - """ - model_stat_file_dir = os.path.join( - job_dict['DATA'], job_dict['VERIF_CASE']+'_'+job_dict['STEP'], - 'METplus_output', job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'] - ) - stat_file_list = glob.glob(os.path.join(model_stat_file_dir, '*.stat')) - if len(stat_file_list) != 0: - stat_files_exist = True - else: - stat_files_exist = False - return stat_files_exist - - -def check_pstat_files(job_dict): - """! Check for MET point_stat files - - Args: - job_dict - dictionary containing settings - job is running with (strings) - - Returns: - pstat_files_exist - if point_stat files - exist or not (boolean) - """ - FHR_START = os.environ ['FHR_START'] - FHR_START2= str(FHR_START).zfill(2) - model_stat_file_dir = os.path.join( - job_dict['DATA'], job_dict['VERIF_CASE'], - 'METplus_output', job_dict['VERIF_TYPE'],'point_stat', - job_dict['MODEL']+'.'+job_dict['VDATE'] - ) - - pstat_file = os.path.join( - model_stat_file_dir, 'point_stat_'+job_dict['MODEL']+ - '_'+job_dict['NEST']+'_'+job_dict['VAR_NAME']+ - '_OBS_*0000L_'+job_dict['VDATE']+'_'+job_dict['VHOUR']+'0000V.stat' - ) - - stat_file_list = glob.glob(pstat_file) - - if len(stat_file_list) != 0: - pstat_files_exist = True - else: - pstat_files_exist = False - return pstat_files_exist - - - -def format_filler(unfilled_file_format, valid_time_dt, init_time_dt, - forecast_hour, str_sub_dict): - """! Creates a filled file path from a format - Args: - unfilled_file_format - file naming convention (string) - valid_time_dt - valid time (datetime) - init_time_dt - initialization time (datetime) - forecast_hour - forecast hour (string) - str_sub_dict - other strings to substitue (dictionary) - Returns: - filled_file_format - file_format filled in with verifying - time information (string) - """ - filled_file_format = '/' - format_opt_list = ['lead', 'lead_shift', 'valid', 'valid_shift', - 'init', 'init_shift', 'cycle'] - if len(list(str_sub_dict.keys())) != 0: - format_opt_list = format_opt_list+list(str_sub_dict.keys()) - for filled_file_format_chunk in unfilled_file_format.split('/'): - for format_opt in format_opt_list: - nformat_opt = ( - filled_file_format_chunk.count('{'+format_opt+'?fmt=') - ) - if nformat_opt > 0: - format_opt_count = 1 - while format_opt_count <= nformat_opt: - if format_opt in ['lead_shift', 'valid_shift', - 'init_shift']: - shift = (filled_file_format_chunk \ - .partition('shift=')[2] \ - .partition('}')[0]) - format_opt_count_fmt = ( - filled_file_format_chunk \ - .partition('{'+format_opt+'?fmt=')[2] \ - .rpartition('?')[0] - ) - else: - format_opt_count_fmt = ( - filled_file_format_chunk \ - .partition('{'+format_opt+'?fmt=')[2] \ - .partition('}')[0] - ) - if format_opt == 'valid': - replace_format_opt_count = valid_time_dt.strftime( - format_opt_count_fmt - ) - elif format_opt == 'lead': - if format_opt_count_fmt == '%1H': - if int(forecast_hour) < 10: - replace_format_opt_count = forecast_hour[1] - else: - replace_format_opt_count = forecast_hour - elif format_opt_count_fmt == '%2H': - replace_format_opt_count = forecast_hour.zfill(2) - elif format_opt_count_fmt == '%3H': - replace_format_opt_count = forecast_hour.zfill(3) - else: - replace_format_opt_count = forecast_hour - elif format_opt == 'init': - replace_format_opt_count = init_time_dt.strftime( - format_opt_count_fmt - ) - elif format_opt == 'cycle': - replace_format_opt_count = init_time_dt.strftime( - format_opt_count_fmt - ) - elif format_opt == 'lead_shift': - shift = (filled_file_format_chunk.partition('shift=')[2]\ - .partition('}')[0]) - forecast_hour_shift = str(int(forecast_hour) - + int(shift)) - if format_opt_count_fmt == '%1H': - if int(forecast_hour_shift) < 10: - replace_format_opt_count = ( - forecast_hour_shift[1] - ) - else: - replace_format_opt_count = forecast_hour_shift - elif format_opt_count_fmt == '%2H': - replace_format_opt_count = ( - forecast_hour_shift.zfill(2) - ) - elif format_opt_count_fmt == '%3H': - replace_format_opt_count = ( - forecast_hour_shift.zfill(3) - ) - else: - replace_format_opt_count = forecast_hour_shift - elif format_opt == 'init_shift': - shift = (filled_file_format_chunk.partition('shift=')[2]\ - .partition('}')[0]) - init_shift_time_dt = ( - init_time_dt + td(hours=int(shift)) - ) - replace_format_opt_count = init_shift_time_dt.strftime( - format_opt_count_fmt - ) - elif format_opt == 'valid_shift': - shift = (filled_file_format_chunk.partition('shift=')[2]\ - .partition('}')[0]) - valid_shift_time_dt = ( - valid_time_dt + td(hours=int(shift)) - ) - replace_format_opt_count = valid_shift_time_dt.strftime( - format_opt_count_fmt - ) - else: - replace_format_opt_count = str_sub_dict[format_opt] - if format_opt in ['lead_shift', 'valid_shift', 'init_shift']: - filled_file_format_chunk = ( - filled_file_format_chunk.replace( - '{'+format_opt+'?fmt=' - +format_opt_count_fmt - +'?shift='+shift+'}', - replace_format_opt_count - ) - ) - else: - filled_file_format_chunk = ( - filled_file_format_chunk.replace( - '{'+format_opt+'?fmt=' - +format_opt_count_fmt+'}', - replace_format_opt_count - ) - ) - format_opt_count+=1 - filled_file_format = os.path.join(filled_file_format, - filled_file_format_chunk) - return filled_file_format - -def initalize_job_env_dict(): - """! This initializes a dictionary of environment variables and their - values to be set for the job pulling from environment variables - already set previously - Args: - - Returns: - job_env_dict - dictionary of job settings - """ - os.environ['MET_TMP_DIR'] = os.path.join(os.environ['DATA'], 'tmp') - job_env_var_list = [ - 'machine', 'evs_ver', 'HOMEevs', 'FIXevs', 'USHevs', 'DATA', - 'NET', 'RUN', 'VERIF_CASE', 'STEP', 'COMPONENT', 'evs_run_mode', - 'COMROOT', 'COMIN', 'COMOUT', 'COMOUTsmall', 'COMOUTfinal', 'EVSIN', - 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', - 'MET_ROOT', - 'MET_TMP_DIR', 'MODELNAME', 'JOB_GROUP' - ] - job_env_dict = {} - for env_var in job_env_var_list: - job_env_dict[env_var] = os.environ[env_var] - if env_var in ['LOG_MET_OUTPUT_TO_METPLUS', - ]: - job_env_dict[env_var.lower()] = os.environ[env_var] - return job_env_dict - -def metplus_command(conf_file_name): - """! Write out full call to METplus - Args: - conf_file_name - METplus conf file name (string) - Returns: - metplus_cmd - full call to METplus (string) - """ - run_metplus = os.path.join(os.environ['METPLUS_PATH'], 'ush', - 'run_metplus.py') - machine_conf = os.path.join(os.environ['PARMevs'], 'metplus_config', - 'machine.conf') - conf_file = os.path.join(os.environ['PARMevs'], 'metplus_config', os.environ['STEP'], - os.environ['COMPONENT'], os.environ['VERIF_CASE'], - conf_file_name) - if not os.path.exists(conf_file): - print("ERROR: "+conf_file+" DOES NOT EXIST") - sys.exit(1) - metplus_cmd = run_metplus+' -c '+machine_conf+' -c '+conf_file - return metplus_cmd - -def precip_check_obs_input_output_files(job_dict): - """! Check precip observation input and output files - in COMOUT and DATA - Args: - job_dict - job dictionary - Returns: - all_input_file_exist - if all expected - input files exist - (boolean) - input_files_list - list of input files - (strings) - all_COMOUT_file_exist - if all expected - output COMOUT files - exist (boolean) - COMOUT_files_list - list of output COMOUT - files (strings) - DATA_files_list - list of output DATA - files (strings) - """ - valid_date_dt = datetime.strptime( - job_dict['DATE']+job_dict['valid_hour_start'], - '%Y%m%d%H' - ) - # Expected input file - input_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - if job_dict['job_name'] in ['24hrCCPA', '03hrCCPA', '01hrCCPA']: - nccpa_files = ( - int(job_dict['accum']) - /int(job_dict['ccpa_file_accum']) - ) - n = 1 - while n <= nccpa_files: - nccpa_file = os.path.join( - job_dict['DATA'], 'data', 'ccpa', - f"ccpa.accum{job_dict['ccpa_file_accum'].zfill(2)}hr.v" - +(valid_date_dt - -td(hours=(n-1) - *int(job_dict['ccpa_file_accum'])))\ - .strftime('%Y%m%d%H') - ) - input_files_list.append(nccpa_file) - n+=1 - elif job_dict['JOB_GROUP'] == 'generate_stats': - if job_dict['obs'] == 'ccpa': - input_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - "pcp_combine_ccpa_accum" - +f"{job_dict['accum']}hr_valid" - +f"{valid_date_dt:%Y%m%d%H}.nc") - ) - elif job_dict['obs'] == 'mrms': - input_files_list.append( - os.path.join(job_dict['DATA'], 'data', job_dict['obs'], - f"{job_dict['area']}_MultiSensor_QPE_" - +f"{job_dict['accum']}H_Pass2_00.00_" - +f"{valid_date_dt:%Y%m%d}-" - +f"{valid_date_dt:%H%M%S}.grib2") - ) - input_files_exist_list = [] - for input_file in input_files_list: - if check_file(input_file): - input_files_exist_list.append(True) - else: - input_files_exist_list.append(False) - if all(x == True for x in input_files_exist_list) \ - and len(input_files_exist_list) > 0: - all_input_file_exist = True - else: - all_input_file_exist = False - # Expected output files (in COMOUT and DATA) - COMOUT_files_list = [] - DATA_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - if job_dict['job_name'] in ['24hrCCPA', '03hrCCPA', '01hrCCPA']: - file_name = ("pcp_combine_ccpa_accum" - +f"{job_dict['accum']}hr_valid" - +f"{valid_date_dt:%Y%m%d%H}.nc") - COMOUT_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - DATA_files_list.append( - os.path.join(job_dict['DATA'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - COMOUT_files_exist_list = [] - for COMOUT_file in COMOUT_files_list: - if check_file(COMOUT_file): - COMOUT_files_exist_list.append(True) - else: - COMOUT_files_exist_list.append(False) - if all(x == True for x in COMOUT_files_exist_list) \ - and len(COMOUT_files_exist_list) > 0: - all_COMOUT_file_exist = True - else: - all_COMOUT_file_exist = False - return (all_input_file_exist, input_files_list, \ - all_COMOUT_file_exist, COMOUT_files_list, - DATA_files_list) - - -def precip_check_model_input_output_files(job_dict): - """! Check precip model input and output files - in COMOUT and DATA - Args: - job_dict - job dictionary - Returns: - all_input_file_exist - if all expected - input files exist - (boolean) - input_files_list - list of input files - (strings) - all_COMOUT_file_exist - if all expected - output COMOUT files - exist (boolean) - COMOUT_files_list - list of output COMOUT - files (strings) - DATA_files_list - list of output DATA - files (strings) - """ - valid_date_dt = datetime.strptime( - job_dict['DATE']+job_dict['valid_hour_start'], - '%Y%m%d%H' - ) - init_date_dt = (valid_date_dt - - td(hours=int(job_dict['fcst_hour']))) - # Expected input file - input_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - if job_dict['pcp_combine_method'] == 'SUBTRACT': - for fhr in [job_dict['fcst_hour'], - str(int(job_dict['fcst_hour']) - - int(job_dict['accum']))]: - input_files_list.append( - os.path.join(job_dict['DATA'], 'data', - job_dict['MODELNAME'], - f"{job_dict['MODELNAME']}." - +f"{job_dict['area']}." - +f"init{init_date_dt:%Y%m%d%H}." - +f"f{fhr.zfill(3)}") - ) - elif job_dict['pcp_combine_method'] == 'SUM': - naccum_files = int(job_dict['accum'])/int(job_dict['input_accum']) - n = 1 - while n <= naccum_files: - naccum_file = os.path.join( - job_dict['DATA'], 'data', job_dict['MODELNAME'], - f"{job_dict['MODELNAME']}.{job_dict['area']}." - +f"init{init_date_dt:%Y%m%d%H}.f" - +str(int(job_dict['fcst_hour']) - -((n-1)*int(job_dict['input_accum']))).zfill(3) - ) - input_files_list.append(naccum_file) - n+=1 - elif job_dict['JOB_GROUP'] == 'generate_stats': - input_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - f"pcp_combine_{job_dict['MODELNAME']}_" - +f"accum{job_dict['accum']}hr_" - +f"{job_dict['area']}_" - +f"init{init_date_dt:%Y%m%d%H}_" - +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") - ) - input_files_exist_list = [] - for input_file in input_files_list: - if check_file(input_file): - input_files_exist_list.append(True) - else: - input_files_exist_list.append(False) - if all(x == True for x in input_files_exist_list) \ - and len(input_files_exist_list) > 0: - all_input_file_exist = True - else: - all_input_file_exist = False - # Expected output files (in COMOUT and DATA) - COMOUT_files_list = [] - DATA_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - file_name = (f"pcp_combine_{job_dict['MODELNAME']}_" - +f"accum{job_dict['accum']}hr_" - +f"{job_dict['area']}_" - +f"init{init_date_dt:%Y%m%d%H}_" - +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") - COMOUT_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - DATA_files_list.append( - os.path.join(job_dict['DATA'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - elif job_dict['JOB_GROUP'] == 'generate_stats': - file_name = (f"grid_stat_{job_dict['job_name']}_" - f"{job_dict['fcst_hour'].zfill(2)}0000L_" - +f"{valid_date_dt:%Y%m%d_%H%M%S}V.stat") - COMOUT_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - DATA_files_list.append( - os.path.join(job_dict['DATA'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - COMOUT_files_exist_list = [] - for COMOUT_file in COMOUT_files_list: - if check_file(COMOUT_file): - COMOUT_files_exist_list.append(True) - else: - COMOUT_files_exist_list.append(False) - if all(x == True for x in COMOUT_files_exist_list) \ - and len(COMOUT_files_exist_list) > 0: - all_COMOUT_file_exist = True - else: - all_COMOUT_file_exist = False - return (all_input_file_exist, input_files_list, \ - all_COMOUT_file_exist, COMOUT_files_list, - DATA_files_list) - -def snowfall_check_obs_input_output_files(job_dict): - """! Check snowfall observation input and output files - in COMOUT and DATA - Args: - job_dict - job dictionary - Returns: - all_input_file_exist - if all expected - input files exist - (boolean) - input_files_list - list of input files - (strings) - all_COMOUT_file_exist - if all expected - output COMOUT files - exist (boolean) - COMOUT_files_list - list of output COMOUT - files (strings) - DATA_files_list - list of output DATA - files (strings) - """ - valid_date_dt = datetime.strptime( - job_dict['DATE']+job_dict['valid_hour_start'], - '%Y%m%d%H' - ) - # Expected input file - input_files_list = [] - if job_dict['JOB_GROUP'] == 'generate_stats': - if job_dict['obs'] == 'nohrsc': - input_files_list.append( - os.path.join(job_dict['DATA'], 'data', 'nohrsc', - f"nohrsc.accum{job_dict['accum']}hr." - +f"v{valid_date_dt:%Y%m%d%H}") - ) - input_files_exist_list = [] - for input_file in input_files_list: - if check_file(input_file): - input_files_exist_list.append(True) - else: - input_files_exist_list.append(False) - if all(x == True for x in input_files_exist_list) \ - and len(input_files_exist_list) > 0: - all_input_file_exist = True - else: - all_input_file_exist = False - # Expected output files (in COMOUT and DATA) - COMOUT_files_list = [] - DATA_files_list = [] - # - COMOUT_files_exist_list = [] - for COMOUT_file in COMOUT_files_list: - if check_file(COMOUT_file): - COMOUT_files_exist_list.append(True) - else: - COMOUT_files_exist_list.append(False) - if all(x == True for x in COMOUT_files_exist_list) \ - and len(COMOUT_files_exist_list) > 0: - all_COMOUT_file_exist = True - else: - all_COMOUT_file_exist = False - return (all_input_file_exist, input_files_list, \ - all_COMOUT_file_exist, COMOUT_files_list, - DATA_files_list) - -def snowfall_check_model_input_output_files(job_dict): - """! Check snowfall model input and output files - in COMOUT and DATA - Args: - job_dict - job dictionary - Returns: - all_input_file_exist - if all expected - input files exist - (boolean) - input_files_list - list of input files - (strings) - all_COMOUT_file_exist - if all expected - output COMOUT files - exist (boolean) - COMOUT_files_list - list of output COMOUT - files (strings) - DATA_files_list - list of output DATA - files (strings) - """ - valid_date_dt = datetime.strptime( - job_dict['DATE']+job_dict['valid_hour_start'], - '%Y%m%d%H' - ) - init_date_dt = (valid_date_dt - - td(hours=int(job_dict['fcst_hour']))) - # Expected input file - input_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - if job_dict['pcp_combine_method'] in ['SUBTRACT', 'USER_DEFINED']: - for fhr in [job_dict['fcst_hour'], - str(int(job_dict['fcst_hour']) - - int(job_dict['accum']))]: - input_files_list.append( - os.path.join(job_dict['DATA'], 'data', - job_dict['MODELNAME'], - f"{job_dict['MODELNAME']}." - +f"init{init_date_dt:%Y%m%d%H}." - +f"f{fhr.zfill(3)}") - ) - elif job_dict['JOB_GROUP'] == 'generate_stats': - input_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - f"pcp_combine_{job_dict['MODELNAME']}_" - +f"accum{job_dict['accum']}hr_" - +f"{job_dict['snow_var']}_" - +f"init{init_date_dt:%Y%m%d%H}_" - +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") - ) - input_files_exist_list = [] - for input_file in input_files_list: - if check_file(input_file): - input_files_exist_list.append(True) - else: - input_files_exist_list.append(False) - if all(x == True for x in input_files_exist_list) \ - and len(input_files_exist_list) > 0: - all_input_file_exist = True - else: - all_input_file_exist = False - # Expected output files (in COMOUT and DATA) - COMOUT_files_list = [] - DATA_files_list = [] - if job_dict['JOB_GROUP'] == 'assemble_data': - file_name = (f"pcp_combine_{job_dict['MODELNAME']}_" - +f"accum{job_dict['accum']}hr_" - +f"{job_dict['snow_var']}_" - +f"init{init_date_dt:%Y%m%d%H}_" - +f"fhr{job_dict['fcst_hour'].zfill(3)}.nc") - COMOUT_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - DATA_files_list.append( - os.path.join(job_dict['DATA'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - elif job_dict['JOB_GROUP'] == 'generate_stats': - file_name = (f"grid_stat_{job_dict['job_name']}_" - f"{job_dict['fcst_hour'].zfill(2)}0000L_" - +f"{valid_date_dt:%Y%m%d_%H%M%S}V.stat") - COMOUT_files_list.append( - os.path.join(job_dict['COMOUT'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - DATA_files_list.append( - os.path.join(job_dict['DATA'], - f"{job_dict['RUN']}.{valid_date_dt:%Y%m%d}", - job_dict['MODELNAME'], job_dict['VERIF_CASE'], - file_name) - ) - COMOUT_files_exist_list = [] - for COMOUT_file in COMOUT_files_list: - if check_file(COMOUT_file): - COMOUT_files_exist_list.append(True) - else: - COMOUT_files_exist_list.append(False) - if all(x == True for x in COMOUT_files_exist_list) \ - and len(COMOUT_files_exist_list) > 0: - all_COMOUT_file_exist = True - else: - all_COMOUT_file_exist = False - return (all_input_file_exist, input_files_list, \ - all_COMOUT_file_exist, COMOUT_files_list, - DATA_files_list) - - -def get_completed_jobs(completed_jobs_file): - completed_jobs = set() - if os.path.exists(completed_jobs_file): - with open(completed_jobs_file, 'r') as f: - completed_jobs = set(f.read().splitlines()) - return completed_jobs - -def mark_job_completed(completed_jobs_file, job_name): - with open(completed_jobs_file, 'a') as f: - f.write(job_name + "\n") - -# Construct a file name given a template -def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", - VDATE="YYYYmmdd", VHOUR="HH", VDATEHOUR="YYYYmmddHH", - VDATEm1H="YYYYmmdd", VDATEHOURm1H="YYYYmmddHH", - FHR="HH", LVL="0", OFFSET="HH"): - template_str = template_str.replace('{IDATE}', IDATE) - template_str = template_str.replace('{IHOUR}', IHOUR) - template_str = template_str.replace('{VDATE}', VDATE) - template_str = template_str.replace('{VHOUR}', VHOUR) - template_str = template_str.replace('{VDATEHOUR}', VDATEHOUR) - template_str = template_str.replace('{VDATEm1H}', VDATEm1H) - template_str = template_str.replace('{VDATEHOURm1H}', VDATEHOURm1H) - template_str = template_str.replace('{FHR}', FHR) - template_str = template_str.replace('{LVL}', LVL) - template_str = template_str.replace('{OFFSET}', OFFSET) - return template_str - -# Create a list of prepbufr file paths -def get_prepbufr_templates(indir, vdates, paths=[], obsname='both', already_preprocessed=False): - ''' - indir - (str) Input directory for prepbufr file data - vdates - (datetime object) List of datetimes used to fill templates - paths - (list of str) list of paths to append the prepbufr paths to - Default is empty. - ''' - prepbufr_templates = [] - prepbufr_paths = [] - for v, vdate in enumerate(vdates): - vh = vdate.strftime('%H') - vd = vdate.strftime('%Y%m%d') - if vh in ['00', '03', '06', '09', '12', '15', '18', '21']: - if vh in ['03', '09', '15', '21']: - offsets = ['03'] - elif vh in ['00', '06', '12', '18']: - offsets = ['00', '06'] - if obsname in ['both', 'raob']: - if not already_preprocessed: - prepbufr_templates.append(os.path.join( - indir, - 'gdas.{VDATE}', - '{VHOUR}', - 'atmos', - 'gdas.t{VHOUR}z.prepbufr' - )) - else: - prepbufr_templates.append(os.path.join( - indir, - 'gdas.t{VHOUR}z.prepbufr' - )) - for offset in offsets: - use_vdate = vdate + td(hours=int(offset)) - use_vd = use_vdate.strftime('%Y%m%d') - use_vh = use_vdate.strftime('%H') - if obsname in ['both', 'metar']: - if not already_preprocessed: - template = os.path.join( - indir, - 'nam.{VDATE}', - 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' - ) - else: - template = os.path.join( - indir, - 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' - ) - prepbufr_paths.append(fname_constructor( - template, VDATE=use_vd, VHOUR=use_vh, OFFSET=offset - )) - for template in prepbufr_templates: - prepbufr_paths.append(fname_constructor( - template, VDATE=vd, VHOUR=vh - )) - return np.concatenate((paths, np.unique(prepbufr_paths))) - -def preprocess_prepbufr(indir, fname, workdir, outdir, subsets): - if os.path.exists(os.path.join(outdir, fname)): - print(f"{fname} exists in {outdir} so we can skip preprocessing.") - else: - wd = os.getcwd() - os.chdir(workdir) - if os.path.isfile(os.path.join(indir, fname)): - run_shell_command( - [ - os.path.join(os.environ['bufr_ROOT'], 'bin', 'split_by_subset'), - os.path.join(indir, fname) - ] - ) - if all([os.path.isfile(subset) for subset in subsets]): - run_shell_command( - np.concatenate(( - ['cat'], subsets, ['>>', os.path.join(outdir, fname)] - )) - ) - else: - raise FileNotFoundError( - f"The following prepbufr subsets do not exist in {workdir}: " - + ', '.join([subset for subset in subsets if not os.path.isfile(subset)]) - + ". Cannot concatenate subsets." - ) - else: - print( - "WARNING: The following file does not exist: " - + f"{os.path.join(indir, fname)}." - + " Skipping split by subset." - ) - os.chdir(wd) - From be78866cd28921ee464ed1ecc0bd915d409dd659 Mon Sep 17 00:00:00 2001 From: perry shafran Date: Wed, 15 Jan 2025 14:57:10 +0000 Subject: [PATCH 6/8] Updating for rap and completing restart fixes --- .../exevs_mesoscale_rap_grid2obs_stats.sh | 9 +++++++-- ush/mesoscale/mesoscale_util.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh index 535bb7253..61deeae2c 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh @@ -43,6 +43,7 @@ echo "*****************************" # Reformat MET Data export job_type="reformat" export njob=1 + export run_restart=true for NEST in $NEST_LIST; do export NEST=$NEST for VERIF_TYPE in $VERIF_TYPES; do @@ -58,8 +59,12 @@ echo "*****************************" # Check for restart files reformat echo " Check for restart files reformat begin" if [ $evs_run_mode = production ]; then - ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh - export err=$?; err_chk + # Check For Restart Files + if [ "$run_restart" = true ]; then + python ${USHevs}/mesoscale/mesoscale_production_restart.py + export err=$?; err_chk + export run_restart=false + fi fi echo " Check for restart files reformat done" diff --git a/ush/mesoscale/mesoscale_util.py b/ush/mesoscale/mesoscale_util.py index ff0af39e9..a3a306df4 100644 --- a/ush/mesoscale/mesoscale_util.py +++ b/ush/mesoscale/mesoscale_util.py @@ -444,7 +444,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{vx_mask}.{vdate}', )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): copy_files.append(f'{vx_mask}_t{vhour}z_f{str(fhr).zfill(2)}.nc') elif met_tool == 'grid_stat': if verif_case == "snowfall": @@ -463,7 +463,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_{var_name}*{acc}H_{str(verif_type).upper()}_NBRHD{nbrhd}*_' + f'{str(fhr).zfill(2)}0000L_{vdate}_{vhour}0000V.stat' @@ -483,7 +483,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_*_{acc}H_{str(verif_type).upper()}_NBRHD{nbrhd}*_' + f'{str(fhr).zfill(2)}0000L_{vdate}_{vhour}0000V.stat' @@ -507,7 +507,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, model, met_tool, )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') idt = vdt - td(hours=int(fhr)) idate = idt.strftime('%Y%m%d') @@ -551,7 +551,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') idt = vdt - td(hours=int(fhr)) idate = idt.strftime('%Y%m%d') @@ -573,7 +573,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, e = (f"FATAL ERROR: None encountered as an argument while copying" + f" {met_tool} METplus output to COMOUT directory.") raise TypeError(e) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): vdt = datetime.strptime(f'{vdate}{vhour}', '%Y%m%d%H') idt = vdt - td(hours=int(fhr)) idate = idt.strftime('%Y%m%d') @@ -618,7 +618,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_{vx_mask}_{var_name}_OBS*_{str(fhr).zfill(2)}0000L_{vdate}_' + f'{vhour}0000V.stat' @@ -647,7 +647,7 @@ def copy_data_to_restart(data_dir, restart_dir, met_tool=None, net=None, met_tool, f'{model}.{vdate}' )) - for fhr in np.arange(int(fhr_start), int(fhr_end), int(fhr_incr)): + for fhr in np.arange(int(fhr_start), int(fhr_end) + int(fhr_incr), int(fhr_incr)): copy_files.append( f'{met_tool}_{model}_t{vhour}z_{verif_type}_{vx_mask}_job{njob}_' + f'fhr{str(fhr).zfill(2)}.nc' From 87be9652300035fb408a6d12ccf0bcf4c87c4874 Mon Sep 17 00:00:00 2001 From: perry shafran Date: Thu, 16 Jan 2025 17:25:59 +0000 Subject: [PATCH 7/8] Keeping current develop version of meoscale_production_restart.py --- ush/mesoscale/mesoscale_production_restart.py | 38 ++++++------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/ush/mesoscale/mesoscale_production_restart.py b/ush/mesoscale/mesoscale_production_restart.py index 3ed7a90da..b19466cb3 100644 --- a/ush/mesoscale/mesoscale_production_restart.py +++ b/ush/mesoscale/mesoscale_production_restart.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 -# ============================================================================= -# -# NAME: mesoscale_production_restart.py -# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB -# PURPOSE: Check the appropriate restart directory for restart files and copy -# the available files to the working directory -# -# ============================================================================= +''' +Name: mesoscale_production_restart.py +Contact(s): Marcel Caron +Abstract: +''' import os import glob @@ -23,33 +20,20 @@ NET = os.environ['NET'] RUN = os.environ['RUN'] COMPONENT = os.environ['COMPONENT'] -STEP = os.environ['STEP'] VERIF_CASE = os.environ['VERIF_CASE'] +STEP = os.environ['STEP'] # Copy files for restart -if STEP == 'stats': - VERIF_CASE = os.environ['VERIF_CASE'] - RESTART_DIR = os.environ['RESTART_DIR'] - working_dir = os.path.join(DATA, VERIF_CASE) - completed_jobs_file = os.path.join(RESTART_DIR, 'completed_jobs.txt') - if os.path.exists(RESTART_DIR): - if (os.path.exists(completed_jobs_file) - and os.stat(completed_jobs_file).st_size != 0): - print(f"Copying restart directory {RESTART_DIR} " - +f"into working directory {working_dir}") - cutil.run_shell_command( - ['cp', '-rpv', RESTART_DIR, working_dir] - ) -elif STEP == 'plots': +if STEP == 'plots': COMOUTplots = os.environ['COMOUTplots'] RESTART_DIR = os.environ['RESTART_DIR'] - SAVE_DIR = os.environ['SAVE_DIR'] - completed_jobs_file = os.path.join(RESTART_DIR, f'completed_jobs.txt') + COMPLETED_JOBS_FILE = os.environ['COMPLETED_JOBS_FILE'] + working_dir = os.path.join(DATA, VERIF_CASE, 'out') + completed_jobs_file = os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE) if os.path.exists(completed_jobs_file): if os.stat(completed_jobs_file).st_size != 0: cutil.run_shell_command( - ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), SAVE_DIR] + ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), working_dir] ) - print("END: "+os.path.basename(__file__)) From 2d104ad70cc370c832ba68cb3791e76ee7b6139d Mon Sep 17 00:00:00 2001 From: perry shafran Date: Fri, 17 Jan 2025 16:28:13 +0000 Subject: [PATCH 8/8] Restoring the updated mesoscale_production_restart.py file --- ush/mesoscale/mesoscale_production_restart.py | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/ush/mesoscale/mesoscale_production_restart.py b/ush/mesoscale/mesoscale_production_restart.py index b19466cb3..3ed7a90da 100644 --- a/ush/mesoscale/mesoscale_production_restart.py +++ b/ush/mesoscale/mesoscale_production_restart.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -''' -Name: mesoscale_production_restart.py -Contact(s): Marcel Caron -Abstract: -''' +# ============================================================================= +# +# NAME: mesoscale_production_restart.py +# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# PURPOSE: Check the appropriate restart directory for restart files and copy +# the available files to the working directory +# +# ============================================================================= import os import glob @@ -20,20 +23,33 @@ NET = os.environ['NET'] RUN = os.environ['RUN'] COMPONENT = os.environ['COMPONENT'] -VERIF_CASE = os.environ['VERIF_CASE'] STEP = os.environ['STEP'] +VERIF_CASE = os.environ['VERIF_CASE'] # Copy files for restart -if STEP == 'plots': +if STEP == 'stats': + VERIF_CASE = os.environ['VERIF_CASE'] + RESTART_DIR = os.environ['RESTART_DIR'] + working_dir = os.path.join(DATA, VERIF_CASE) + completed_jobs_file = os.path.join(RESTART_DIR, 'completed_jobs.txt') + if os.path.exists(RESTART_DIR): + if (os.path.exists(completed_jobs_file) + and os.stat(completed_jobs_file).st_size != 0): + print(f"Copying restart directory {RESTART_DIR} " + +f"into working directory {working_dir}") + cutil.run_shell_command( + ['cp', '-rpv', RESTART_DIR, working_dir] + ) +elif STEP == 'plots': COMOUTplots = os.environ['COMOUTplots'] RESTART_DIR = os.environ['RESTART_DIR'] - COMPLETED_JOBS_FILE = os.environ['COMPLETED_JOBS_FILE'] - working_dir = os.path.join(DATA, VERIF_CASE, 'out') - completed_jobs_file = os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE) + SAVE_DIR = os.environ['SAVE_DIR'] + completed_jobs_file = os.path.join(RESTART_DIR, f'completed_jobs.txt') if os.path.exists(completed_jobs_file): if os.stat(completed_jobs_file).st_size != 0: cutil.run_shell_command( - ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), working_dir] + ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), SAVE_DIR] ) + print("END: "+os.path.basename(__file__))