singularity_create_aquaplanet.sh
#!/bin/bash -fe
# E3SM Water Cycle v2 run_e3sm script template.
#
# Inspired by v1 run_e3sm script as well as SCREAM group simplified run script.
#
# Bash coding style inspired by:
# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming
# TO DO:
# - custom pelayout
# --------------------------------------------------------------------------
#
# JH: This is a modified version of the run template written for testing the
# E3SM install on GreatLakes. Changes are marked with a JH
#
# --------------------------------------------------------------------------
main() {
# For debugging, uncomment line below
#set -x
# --- Configuration flags ----
# Machine and project
readonly MACHINE=singularity_gl
readonly PROJECT="e3sm"
# Simulation
# JH: We do not want to test the full coupled model here, rather we just want to ensure
# that we can run compsets with active EAM. Uncomment these lines for the fully-coupled
# CMIP6 configuration
#readonly COMPSET="WCYCL1850"
#readonly RESOLUTION="ne30pg2_EC30to60E2r2"
#readonly CASE_NAME="v2.LR.piControl"
#readonly CASE_GROUP="v2.LR"
readonly COMPSET="F-EAM-AQP1"
readonly RESOLUTION="ne4pg2_ne4pg2"
readonly CASE_NAME=${COMPSET}"_"${RESOLUTION}
# Code and compilation
# JH: CHECKOUT and BRANCH are not used and left as they are; if you are
# running this, either change do_fetch_code to true and change these
# parameters, or clone the code yourself and update the paths below
readonly CHECKOUT="20210806"
readonly BRANCH="master"
readonly CHERRY=( )
readonly DEBUG_COMPILE=false
# Run options
# JH: we make this 'initial', in which case we need to provide ncdata to user_nl_eam...
readonly MODEL_START_TYPE="initial" # 'initial', 'continue', 'branch', 'hybrid'
readonly START_DATE="0001-01-01"
# Additional options for 'branch' and 'hybrid'
# JH: also not used as long as MODEL_START_TYPE=initial. If wanting to use this
# refcase with a 'branch' or 'hybrid' MODEL_START_TYPE, it will need to be downloaded
# somewhere/somehow, and these paths updated
readonly GET_REFCASE=TRUE
readonly RUN_REFDIR="/global/cscratch1/sd/forsyth/E3SMv2/v2.LR.piControl/init"
readonly RUN_REFCASE="20210625.v2rc3c-GWD.piControl.ne30pg2_EC30to60E2r2.chrysalis"
readonly RUN_REFDATE="1001-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid'
# Set paths
# JH: update these to point to your clone of the repository. Uncomment the lines below
# if you are setting do_fetch_code=true and want the script to handle everything
#readonly CODE_ROOT="${HOME}/E3SMv2/code/${CHECKOUT}"
#readonly CASE_ROOT="/global/cscratch1/sd/${USER}/E3SMv2/${CASE_NAME}"
fail() {
echo "ctrl-f for 'fail' in order to find code to change!'
exit 1
}
fail
readonly CODE_ROOT="/home/owhughes/E3SM/CLDERA-E3SM"
readonly CASE_ROOT="/scratch/cjablono_root/cjablono1/owhughes/E3SM/E3SMv2/${CASE_NAME}"
# Sub-directories
readonly CASE_BUILD_DIR=${CASE_ROOT}/build
readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive
# Define type of run
# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays',
# 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays'
# or 'production' for full simulation
# JH: one of the XS options can be selected for simply testing that the model
# installation is working.
# 2x5: simulate for 10 days, as two 5-day runs with a resubmission
# 1x10: simulate for 10 days
# S: 31 nodes
# L: 125 nodes
# XS: maybe 14 nodes, can't find documentation
# M: no idea
readonly run='XS_2x5_ndays'
if [ "${run}" != "production" ]; then
# Short test simulations
tmp=($(echo $run | tr "_" " "))
layout=${tmp[0]}
units=${tmp[2]}
resubmit=$(( ${tmp[1]%%x*} -1 ))
length=${tmp[1]##*x}
readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts
readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run
readonly PELAYOUT=${layout}
readonly WALLTIME="0:30:00"
readonly STOP_OPTION=${units}
readonly STOP_N=${length}
readonly REST_OPTION=${STOP_OPTION}
readonly REST_N=${STOP_N}
readonly RESUBMIT=${resubmit}
readonly DO_SHORT_TERM_ARCHIVING=false
else
# Production simulation
readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts
readonly CASE_RUN_DIR=${CASE_ROOT}/run
readonly PELAYOUT="L"
readonly WALLTIME="34:00:00"
readonly STOP_OPTION="nyears"
readonly STOP_N="50"
readonly REST_OPTION="nyears"
readonly REST_N="5"
readonly RESUBMIT="9"
readonly DO_SHORT_TERM_ARCHIVING=false
fi
# Coupler history
readonly HIST_OPTION="nyears"
readonly HIST_N="5"
# Leave empty (unless you understand what it does)
readonly OLD_EXECUTABLE=""
# --- Toggle flags for what to do ----
# JH: as noted above, this script is currently ready to run with do_fetch_code=false.
# Either edit the paths above to point to your clone of the code, or enable fetching
# by this script
do_fetch_code=false
do_create_newcase=true
do_case_setup=true
do_case_build=true
do_case_submit=false
# --- Now, do the work ---
# Make directories created by this script world-readable
umask 022
# Fetch code from Github
fetch_code
# Create case
create_newcase
# Setup
case_setup
# Build
case_build
# Configure runtime options
runtime_options
# Copy script into case_script directory for provenance
copy_script
# Submit
case_submit
# All done
echo $'\n----- All done -----\n'
}
# =======================
# Custom user_nl settings
# =======================
# JH: the supplied user_nl settings are for the coupled model; we want something more stripped down,
# and are going to need to point to initial conditions from a data file, so we comment these out
# and define a new user_nl_eam
user_nl() {
#cat << EOF >> user_nl_eam
# nhtfrq = 0,-24,-6,-6,-3,-24,0
# mfilt = 1,30,120,120,240,30,1
# avgflag_pertape = 'A','A','I','A','A','A','I'
# fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz'
# fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV'
# fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT'
# fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M'
# fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500'
# fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M'
# fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN'
# fincl7 = 'O3', 'PS', 'TROP_P'
#
#! Additional retuning
# clubb_tk1 = 268.15D0
# gw_convect_hcf = 10.0
#EOF
#
#cat << EOF >> user_nl_elm
# hist_dov2xy = .true.,.true.
# hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA'
# hist_mfilt = 1,365
# hist_nhtfrq = 0,-24
# hist_avgflag_pertape = 'A','A'
#EOF
#
#cat << EOF >> user_nl_mosart
# rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ'
# rtmhist_mfilt = 1,365
# rtmhist_ndens = 2
# rtmhist_nhtfrq = 0,-24
#EOF
# JH: most of these namelist settings, or ones that are similar enough in name, can be looked up
# in the CESM docs for CAM:
# https://www.cesm.ucar.edu/models/cesm2/settings/current/cam_nml.html
cat << EOF >> user_nl_eam
nhtfrq = -12
mfilt = 240
avgflag_pertape = 'A'
fincl1 = 'U', 'V', 'T', 'PS'
! recommended settings for ne30 by https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/1044644202/EAM+s+HOMME+dycore
nu_top = 2.5e5
se_tstep=300
dt_tracer_factor = 6
hypervis_subcycle_q=6
hypervis_scaling=0
nu=1e15
EOF
}
patch_mpas_streams() {
echo
}
######################################################
### Most users won't need to change anything below ###
######################################################
#-----------------------------------------------------
fetch_code() {
if [ "${do_fetch_code,,}" != "true" ]; then
echo $'\n----- Skipping fetch_code -----\n'
return
fi
echo $'\n----- Starting fetch_code -----\n'
local path=${CODE_ROOT}
local repo=e3sm
echo "Cloning $repo repository branch $BRANCH under $path"
if [ -d "${path}" ]; then
echo "ERROR: Directory already exists. Not overwriting"
exit 20
fi
mkdir -p ${path}
pushd ${path}
# This will put repository, with all code
git clone git@github.com:E3SM-Project/${repo}.git .
# Setup git hooks
rm -rf .git/hooks
git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks
git config commit.template .git/hooks/commit.template
# Check out desired branch
git checkout ${BRANCH}
# Custom addition
if [ "${CHERRY}" != "" ]; then
echo ----- WARNING: adding git cherry-pick -----
for commit in "${CHERRY[@]}"
do
echo ${commit}
git cherry-pick ${commit}
done
echo -------------------------------------------
fi
# Bring in all submodule components
git submodule update --init --recursive
popd
}
#-----------------------------------------------------
create_newcase() {
if [ "${do_create_newcase,,}" != "true" ]; then
echo $'\n----- Skipping create_newcase -----\n'
return
fi
echo $'\n----- Starting create_newcase -----\n'
${CODE_ROOT}/cime/scripts/create_newcase \
--case ${CASE_NAME} \
--output-root ${CASE_ROOT} \
--script-root ${CASE_SCRIPTS_DIR} \
--handle-preexisting-dirs u \
--compset ${COMPSET} \
--res ${RESOLUTION} \
--machine ${MACHINE} \
--project ${PROJECT} \
--walltime ${WALLTIME} \
--pecount ${PELAYOUT}\
#--case-group ${CASE_GROUP} JH: don't know what this does, don't need it
if [ $? != 0 ]; then
echo $'\nNote: if create_newcase failed because sub-directory already exists:'
echo $' * delete old case_script sub-directory'
echo $' * or set do_newcase=false\n'
exit 35
fi
}
#-----------------------------------------------------
case_setup() {
if [ "${do_case_setup,,}" != "true" ]; then
echo $'\n----- Skipping case_setup -----\n'
return
fi
echo $'\n----- Starting case_setup -----\n'
pushd ${CASE_SCRIPTS_DIR}
# Setup some CIME directories
./xmlchange EXEROOT=${CASE_BUILD_DIR}
./xmlchange RUNDIR=${CASE_RUN_DIR}
# Short term archiving
./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^}
./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR}
# Build with COSP, except for a data atmosphere (datm)
if [ `./xmlquery --value COMP_ATM` == "datm" ]; then
echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n'
else
echo $'\nConfiguring E3SM to use the COSP simulator\n'
./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp'
fi
# Extracts input_data_dir in case it is needed for user edits to the namelist later
local input_data_dir=`./xmlquery DIN_LOC_ROOT --value`
# Custom user_nl
user_nl
# Finally, run CIME case.setup
./case.setup --reset
popd
}
#-----------------------------------------------------
case_build() {
pushd ${CASE_SCRIPTS_DIR}
# do_case_build = false
if [ "${do_case_build,,}" != "true" ]; then
echo $'\n----- case_build -----\n'
if [ "${OLD_EXECUTABLE}" == "" ]; then
# Ues previously built executable, make sure it exists
if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then
echo 'Skipping build because $do_case_build = '${do_case_build}
else
echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.'
exit 297
fi
else
# If absolute pathname exists and is executable, reuse pre-exiting executable
if [ -x ${OLD_EXECUTABLE} ]; then
echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE}
cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/
else
echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.'
exit 297
fi
fi
echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.'
./xmlchange BUILD_COMPLETE=TRUE
# do_case_build = true
else
echo $'\n----- Starting case_build -----\n'
# Turn on debug compilation option if requested
if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then
./xmlchange DEBUG=${DEBUG_COMPILE^^}
fi
# Run CIME case.build
./case.build
# Some user_nl settings won't be updated to *_in files under the run directory
# Call preview_namelists to make sure *_in and user_nl files are consistent.
./preview_namelists
fi
popd
}
#-----------------------------------------------------
runtime_options() {
echo $'\n----- Starting runtime_options -----\n'
pushd ${CASE_SCRIPTS_DIR}
# Set simulation start date
./xmlchange RUN_STARTDATE=${START_DATE}
# Segment length
./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N}
# Restart frequency
./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N}
# Coupler history
./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N}
# Coupler budgets (always on)
./xmlchange BUDGETS=TRUE
# Set resubmissions
if (( RESUBMIT > 0 )); then
./xmlchange RESUBMIT=${RESUBMIT}
fi
# Run type
# Start from default of user-specified initial conditions
if [ "${MODEL_START_TYPE,,}" == "initial" ]; then
./xmlchange RUN_TYPE="startup"
./xmlchange CONTINUE_RUN="FALSE"
# Continue existing run
elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then
./xmlchange CONTINUE_RUN="TRUE"
elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then
./xmlchange RUN_TYPE=${MODEL_START_TYPE,,}
./xmlchange GET_REFCASE=${GET_REFCASE}
./xmlchange RUN_REFDIR=${RUN_REFDIR}
./xmlchange RUN_REFCASE=${RUN_REFCASE}
./xmlchange RUN_REFDATE=${RUN_REFDATE}
echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE}
echo '$RUN_REFDIR = '${RUN_REFDIR}
echo '$RUN_REFCASE = '${RUN_REFCASE}
echo '$RUN_REFDATE = '${START_DATE}
else
echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.'
exit 380
fi
# Patch mpas streams files
patch_mpas_streams
popd
}
#-----------------------------------------------------
case_submit() {
if [ "${do_case_submit,,}" != "true" ]; then
echo $'\n----- Skipping case_submit -----\n'
return
fi
echo $'\n----- Starting case_submit -----\n'
pushd ${CASE_SCRIPTS_DIR}
# Run CIME case.submit
./case.submit
popd
}
#-----------------------------------------------------
copy_script() {
echo $'\n----- Saving run script for provenance -----\n'
local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance
mkdir -p ${script_provenance_dir}
local this_script_name=`basename $0`
local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S`
cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name}
}
#-----------------------------------------------------
# Silent versions of popd and pushd
pushd() {
command pushd "$@" > /dev/null
}
popd() {
command popd "$@" > /dev/null
}
# Now, actually run the script
#-----------------------------------------------------
main