Merge branch 'NOAA-EMC:develop' into Early_cycle_changes

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -40,6 +40,7 @@ body:
         - WCOSS2
         - Hera
         - Orion
+        - Hercules
         - Jet
         - Cloud
     validations:

.gitignore

@@ -30,8 +30,8 @@ fix/chem
 fix/cice
 fix/cpl
 fix/datm
-fix/gdas
 fix/gldas
+fix/gdas
 fix/gsi
 fix/lut
 fix/mom6
@@ -41,13 +41,12 @@ fix/sfc_climo
 fix/ugwd
 fix/verif
 fix/wave
-fix/wafs
 
 # Ignore parm file symlinks
 #--------------------------
 parm/config/config.base
 parm/gldas
-parm/mon
+parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
@@ -98,8 +97,42 @@ parm/wafs
 #--------------------------------------------
 sorc/*log
 sorc/logs
-sorc/*.cd
-sorc/*.fd
+sorc/calc_analysis.fd
+sorc/calc_increment_ens.fd
+sorc/calc_increment_ens_ncio.fd
+sorc/emcsfc_ice_blend.fd
+sorc/emcsfc_snow2mdl.fd
+sorc/enkf.fd
+sorc/enkf_chgres_recenter_nc.fd
+sorc/fbwndgfs.fd
+sorc/gaussian_sfcanl.fd
+sorc/getsfcensmeanp.fd
+sorc/getsigensmeanp_smooth.fd
+sorc/getsigensstatp.fd
+sorc/gfs_bufr.fd
+sorc/global_cycle.fd
+sorc/gsi.fd
+sorc/interp_inc.fd
+sorc/mkgfsawps.fd
+sorc/overgridid.fd
+sorc/oznmon_horiz.fd
+sorc/oznmon_time.fd
+sorc/radmon_angle.fd
+sorc/radmon_bcoef.fd
+sorc/radmon_bcor.fd
+sorc/radmon_time.fd
+sorc/rdbfmsua.fd
+sorc/recentersigp.fd
+sorc/reg2grb2.fd
+sorc/supvit.fd
+sorc/syndat_getjtbul.fd
+sorc/syndat_maksynrc.fd
+sorc/syndat_qctropcy.fd
+sorc/tave.fd
+sorc/tocsbufr.fd
+sorc/upp.fd
+sorc/vint.fd
+sorc/webtitle.fd
 
 # Ignore scripts from externals
 #------------------------------
@@ -122,6 +155,7 @@ scripts/exgfs_atmos_wafs_grib2_0p25.sh
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
+ush/exglobal_prep_ocean_obs.py
 ush/fix_precip.sh
 ush/fv3gfs_driver_grid.sh
 ush/fv3gfs_filter_topo.sh
@@ -151,7 +185,6 @@ versions/build.ver
 versions/run.ver
 
 # wxflow checkout and symlinks
-sorc/wxflow
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow

.gitmodules

@@ -0,0 +1,28 @@
+[submodule "sorc/ufs_model.fd"]
+	path = sorc/ufs_model.fd
+	url = https://github.com/ufs-community/ufs-weather-model
+	ignore = dirty
+[submodule "sorc/wxflow"]
+	path = sorc/wxflow
+	url = https://github.com/NOAA-EMC/wxflow
+[submodule "sorc/gfs_utils.fd"]
+	path = sorc/gfs_utils.fd
+	url = https://github.com/NOAA-EMC/gfs-utils
+[submodule "sorc/ufs_utils.fd"]
+	path = sorc/ufs_utils.fd
+	url = https://github.com/ufs-community/UFS_UTILS.git
+[submodule "sorc/verif-global.fd"]
+	path = sorc/verif-global.fd
+	url = https://github.com/NOAA-EMC/EMC_verif-global.git
+[submodule "sorc/gsi_enkf.fd"]
+	path = sorc/gsi_enkf.fd
+	url = https://github.com/NOAA-EMC/GSI.git
+[submodule "sorc/gdas.cd"]
+	path = sorc/gdas.cd
+	url = https://github.com/NOAA-EMC/GDASApp.git
+[submodule "sorc/gsi_utils.fd"]
+	path = sorc/gsi_utils.fd
+	url = https://github.com/NOAA-EMC/GSI-Utils.git
+[submodule "sorc/gsi_monitor.fd"]
+	path = sorc/gsi_monitor.fd
+	url = https://github.com/NOAA-EMC/GSI-Monitor.git

README.md

@@ -18,6 +18,7 @@ The `global-workflow` current supports the following tier-1 machines:
 
 * NOAA RDHPCS - Hera
 * MSU HPC - Orion
+* MSU HPC - Hercules
 * NOAA's operational HPC - WCOSS2
 
 Additionally, the following tier-2 machine is supported:

ci/cases/pr/C48_S2SA_gefs.yaml → ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -4,7 +4,7 @@ experiment:
 
 arguments:
   pslot: {{ 'pslot' | getenv }}
-  app: S2SA
+  app: S2SWA
   resdet: 48
   resens: 48
   nens: 2

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -16,3 +16,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - hercules

ci/cases/pr/C96_atm3DVar.yaml

@@ -15,3 +15,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - hercules

ci/platforms/config.hercules

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
+export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+export STMP="/work2/noaa/stmp/${USER}"
+export SLURM_ACCOUNT=nems
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/platforms/config.orion

@@ -1,8 +1,8 @@
 #!/usr/bin/bash
 
-export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT
+export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work/noaa/stmp/${USER}"
+export STMP="/work2/noaa/stmp/${USER}"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/scripts/check_ci.sh

@@ -33,6 +33,7 @@ case ${MACHINE_ID} in
 esac
 set +x
 source "${ROOT_DIR}/ush/module-setup.sh"
+source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
@@ -86,7 +87,7 @@ for pr in ${pr_list}; do
   if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
     sed -i "1 i\`\`\`" "${output_ci}"
-    sed -i "1 i\All CI Test Cases Passed:" "${output_ci}"
+    sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
     "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
@@ -131,8 +132,8 @@ for pr in ${pr_list}; do
       "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
       error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
       {
-       echo "Experiment ${pslot} Terminated: *** FAILED ***"
-       echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
+       echo "Experiment ${pslot}  *** FAILED *** on ${MACHINE_ID^}"
+       echo "Experiment ${pslot}  with ${num_failed} tasks failed at $(date +'%D %r')" || true
        echo "Error logs:"
        echo "${error_logs}"
       } >> "${output_ci}"
@@ -141,7 +142,7 @@ for pr in ${pr_list}; do
       "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
-         sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" |  awk '{print $1}' | xargs scancel || true
+         cancel_slurm_jobs "${pslot}"
       done
       break
     fi
@@ -151,9 +152,9 @@ for pr in ${pr_list}; do
       rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
-      DATE=$(date)
-      echo "Experiment ${pslot} **SUCCESS** ${DATE}" >> "${output_ci_single}"
-      echo "Experiment ${pslot} **SUCCESS** at ${DATE}" >> "${output_ci}"
+      DATE=$(date +'%D %r')
+      echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}"
+      echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
 
     fi

ci/scripts/clone-build_ci.sh

@@ -48,7 +48,7 @@ git clone "${REPO_URL}"
 cd global-workflow || exit 1
 
 # checkout pull request
-"${GH}" pr checkout "${PR}" --repo "${REPO_URL}"
+"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules
 HOMEgfs="${PWD}"
 source "${HOMEgfs}/ush/detect_machine.sh"
 
@@ -67,50 +67,40 @@ source "${HOMEgfs}/ush/detect_machine.sh"
 commit=$(git log --pretty=format:'%h' -n 1)
 echo "${commit}" > "../commit"
 
-# run checkout script
+# build full cycle
 cd sorc || exit 1
 set +e
-./checkout.sh -c -g -u >> log.checkout 2>&1
-checkout_status=$?
-if [[ ${checkout_status} != 0 ]]; then
-  {
-    echo "Checkout: *** FAILED ***"
-    echo "Checkout: Failed at $(date)" || true
-    echo "Checkout: see output at ${PWD}/log.checkout"
-  } >> "${outfile}"
-  exit "${checkout_status}"
-else
-  {
-    echo "Checkout: Completed at $(date)" || true
-  } >> "${outfile}"
-fi
 
-# build full cycle
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh  >> log.build 2>&1
+./build_all.sh -gu  >> log.build 2>&1
 build_status=$?
 
+DATE=$(date +'%D %r')
 if [[ ${build_status} != 0 ]]; then
   {
     echo "Build: *** FAILED ***"
-    echo "Build: Failed at $(date)" || true
-    echo "Build: see output at ${PWD}/log.build"
+    echo "Build: Failed at ${DATE}"
+    cat "${PWD}/log.build"
   } >> "${outfile}"
   exit "${build_status}"
 else
   {
-    echo "Build: Completed at $(date)" || true
+    echo "Build: Completed at ${DATE}"
   } >> "${outfile}"
 fi
 
-./link_workflow.sh
+LINK_LOGFILE_PATH=link_workflow.log
+rm -f "${LINK_LOGFILE_PATH}"
+./link_workflow.sh >> "${LINK_LOGFILE_PATH}" 2>&1
 link_status=$?
 if [[ ${link_status} != 0 ]]; then
+  DATE=$(date +'%D %r')
   {
     echo "Link: *** FAILED ***"
-    echo "Link: Failed at $(date)" || true
+    echo "Link: Failed at ${DATE}"
+    cat "${LINK_LOGFILE_PATH}"
   } >> "${outfile}"
   exit "${link_status}"
 fi