diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 18d015b6e9..a7665189ac 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -12,10 +12,10 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6','hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Workflow Wrapper test depth {0..9}, 0=none, 1=simple, 9=all [default] @@ -103,7 +103,7 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -117,7 +117,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'derecho', 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'gaea-c6', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh index 33fd966efa..d43c828318 100755 --- a/.cicd/scripts/wrapper_srw_ftest.sh +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -28,6 +28,13 @@ if [[ "${SRW_PLATFORM}" == gaea ]]; then sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "gaea" "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh fi +if [[ "${SRW_PLATFORM}" == gaea-c6 ]]; then + sed -i '15i #SBATCH --clusters=c6' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "gaea-c6" "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh +fi + if [[ "${SRW_PLATFORM}" == hera ]]; then if [[ "${SRW_COMPILER}" == gnu ]]; then sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 29a878d4a4..381369d59e 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -32,6 +32,7 @@ - [ ] derecho.intel - [ ] gaea.intel +- [ ] gaea-c6.intel - [ ] hera.gnu - [ ] hera.intel - [ ] hercules.intel diff --git a/Externals.cfg b/Externals.cfg index c3d70aaa3d..0463eb35b4 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 33b3c18 +hash = e119370 local_path = sorc/ufs-weather-model required = True diff --git a/devbuild.sh b/devbuild.sh index 332abb49b5..b61d4505d7 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -285,7 +285,7 @@ set -eu # automatically determine compiler if [ -z "${COMPILER}" ] ; then case ${PLATFORM} in - jet|hera|gaea) COMPILER=intel ;; + jet|hera|gaea|gaea-c6) COMPILER=intel ;; orion) COMPILER=intel ;; wcoss2) COMPILER=intel ;; cheyenne) COMPILER=intel ;; diff --git a/doc/ContribGuide/contributing.rst b/doc/ContribGuide/contributing.rst index 7576d62cfd..30b1a9ccd0 100644 --- a/doc/ContribGuide/contributing.rst +++ b/doc/ContribGuide/contributing.rst @@ -229,6 +229,7 @@ Here is the template that is provided when developers click "Create pull request - [ ] derecho.intel - [ ] gaea.intel + - [ ] gaea-c6.intel - [ ] hera.gnu - [ ] hera.intel - [ ] hercules.intel diff --git a/doc/ContribGuide/testing.rst b/doc/ContribGuide/testing.rst index b296a3f90a..aad606f6fe 100644 --- a/doc/ContribGuide/testing.rst +++ b/doc/ContribGuide/testing.rst @@ -49,9 +49,7 @@ Additionally, users can potentially access the directories where the Jenkins tes * - Derecho - /glade/derecho/scratch/epicufsrt/jenkins/workspace * - Gaea - - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea - * - Gaea C5 - - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea-c5 + - /gpfs/f5/epic/scratch/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea * - Hera (Intel) - /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#__2/hera * - Hera (GNU) diff --git a/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst b/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst index 0607a232b5..7ed30c8c05 100644 --- a/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst +++ b/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst @@ -76,8 +76,8 @@ On most Level 1 systems, a container named ``ubuntu20.04-intel-ue-1.4.1-srw-dev. - File Location * - Derecho [#fn]_ - /glade/work/epicufsrt/contrib/containers - * - Gaea [#fn]_ - - /lustre/f2/dev/role.epic/containers + * - Gaea-C5 [#fn]_ + - /gpfs/f5/epic/world-shared/containers * - Hera - /scratch1/NCEPDEV/nems/role.epic/containers * - Jet diff --git a/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst b/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst index 8f31dd9b1a..bb4272c35c 100644 --- a/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst +++ b/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst @@ -42,7 +42,7 @@ For a detailed explanation of how to build and run the SRW App on any supported ./devbuild.sh --platform= - where ```` is replaced with the name of the user's platform/system. Valid values include: ``derecho`` | ``gaea`` | ``hera`` | ``hercules`` | ``jet`` | ``linux`` | ``macos`` | ``noaacloud`` | ``orion`` | ``wcoss2`` + where ```` is replaced with the name of the user's platform/system. Valid values include: ``derecho`` | ``gaea`` | ``gaea-c6`` | ``hera`` | ``hercules`` | ``jet`` | ``linux`` | ``macos`` | ``noaacloud`` | ``orion`` | ``wcoss2`` For additional details, see :numref:`Section %s `, or view :numref:`Section %s ` to try the CMake build approach instead. @@ -72,7 +72,7 @@ For a detailed explanation of how to build and run the SRW App on any supported Users will need to open the ``config.yaml`` file and adjust the experiment parameters in it to suit the needs of their experiment (e.g., date, grid, physics suite). At a minimum, users need to modify the ``MACHINE`` parameter. In most cases, users will need to specify the ``ACCOUNT`` parameter and the location of the experiment data (see :numref:`Section %s ` for Level 1 system default locations). - For example, a user on Gaea might adjust or add the following fields to run the 12-hr "out-of-the-box" case on Gaea using prestaged system data and :term:`cron` to automate the workflow: + For example, a user on Gaea-C5 might adjust or add the following fields to run the 12-hr "out-of-the-box" case on Gaea-C5 using prestaged system data and :term:`cron` to automate the workflow: .. code-block:: console diff --git a/doc/UsersGuide/BuildingRunningTesting/RunSRW.rst b/doc/UsersGuide/BuildingRunningTesting/RunSRW.rst index b66b399652..b1dea8d7f2 100644 --- a/doc/UsersGuide/BuildingRunningTesting/RunSRW.rst +++ b/doc/UsersGuide/BuildingRunningTesting/RunSRW.rst @@ -55,8 +55,10 @@ The SRW App requires input files to run. These include static datasets, initial - File location * - Derecho - /glade/work/epicufsrt/contrib/UFS_SRW_data/|data|/input_model_data - * - Gaea + * - Gaea-C5 - /gpfs/f5/epic/world-shared/UFS_SRW_data/|data|/input_model_data/ + * - Gaea-C6 + - /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/|data|/input_model_data/ * - Hera - /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/|data|/input_model_data/ * - Hercules diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index 447373cbea..182a3c9c2b 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -30,7 +30,7 @@ If non-default parameters are selected for the variables in this section, they s Setting ``RUN_ENVIR`` to "community" is recommended in most cases for users who are not running in NCO's production environment. Valid values: ``"nco"`` | ``"community"`` ``MACHINE``: (Default: "BIG_COMPUTER") - The machine (a.k.a. platform or system) on which the workflow will run. Currently supported platforms are listed on the :srw-wiki:`SRW App Wiki page `. When running the SRW App on any ParallelWorks/NOAA Cloud system, use "NOAACLOUD" regardless of the underlying system (AWS, GCP, or Azure). Valid values: ``"HERA"`` | ``"ORION"`` | ``"HERCULES"`` | ``"JET"`` | ``"CHEYENNE"`` | ``"DERECHO"`` | ``"GAEA"`` | ``"NOAACLOUD"`` | ``"STAMPEDE"`` | ``"ODIN"`` | ``"MACOS"`` | ``"LINUX"`` | ``"SINGULARITY"`` | ``"WCOSS2"`` (Check ``ufs-srweather-app/ush/valid_param_vals.yaml`` for the most up-to-date list of supported platforms.) + The machine (a.k.a. platform or system) on which the workflow will run. Currently supported platforms are listed on the :srw-wiki:`SRW App Wiki page `. When running the SRW App on any ParallelWorks/NOAA Cloud system, use "NOAACLOUD" regardless of the underlying system (AWS, GCP, or Azure). Valid values: ``"HERA"`` | ``"ORION"`` | ``"HERCULES"`` | ``"JET"`` | ``"CHEYENNE"`` | ``"DERECHO"`` | ``"GAEA"`` | ``"GAEA-C6"`` | ``"NOAACLOUD"`` | ``"STAMPEDE"`` | ``"ODIN"`` | ``"MACOS"`` | ``"LINUX"`` | ``"SINGULARITY"`` | ``"WCOSS2"`` (Check ``ufs-srweather-app/ush/valid_param_vals.yaml`` for the most up-to-date list of supported platforms.) .. hint:: Users who are NOT on a named, supported Level 1 or 2 platform will need to set the ``MACHINE`` variable to ``LINUX`` or ``MACOS``. To combine use of a Linux or MacOS platform with the Rocoto workflow manager, users will also need to set ``WORKFLOW_MANAGER: "rocoto"`` in the ``platform:`` section of ``config.yaml``. This combination will assume a Slurm batch manager when generating the XML. diff --git a/doc/tables/Tests.csv b/doc/tables/Tests.csv index 325b4dad69..eb1691f43d 100644 --- a/doc/tables/Tests.csv +++ b/doc/tables/Tests.csv @@ -52,8 +52,8 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_ ,yes,pregen_grid_orog_sfc_climo,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,17, ,yes,specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2021061500,6,6,19, ,yes,specify_template_filenames,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,28, -,hera/jet/orion/gaea,get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems,RRFS_CONUS_3km,FV3_HRRR,GEFS,GEFS,2022040400,6,805,45, -,hera/jet/orion/gaea,get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,*2 days ago*,6,12,40,Retrieves online data and runs a forecast valid 00z 2 days previous to the test date +,hera/jet/orion/gaea/gaea-c6,get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems,RRFS_CONUS_3km,FV3_HRRR,GEFS,GEFS,2022040400,6,805,45, +,hera/jet/orion/gaea/gaea-c6,get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,*2 days ago*,6,12,40,Retrieves online data and runs a forecast valid 00z 2 days previous to the test date ,hera/jet,custom_ESGgrid_Great_Lakes_snow_8km,*custom*,FV3_RAP,RAP,RAP,2023021700,6,,, ,hera/jet,get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019061200,6,6,19, ,hera/jet,get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019061200,6,10,23, diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index af79ad8a70..6af4f8dd1f 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -40,6 +40,9 @@ else if ( "$L_MACHINE" == singularity ) then else if ( "$L_MACHINE" == gaea ) then module reset +else if ( "$L_MACHINE" == gaea-c6 ) then + module reset + else if ( "$L_MACHINE" == derecho ) then module purge diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index b030d2a9f5..5012d4381c 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -47,6 +47,9 @@ elif [ "$L_MACHINE" = singularity ]; then elif [ "$L_MACHINE" = gaea ]; then module reset +elif [ "$L_MACHINE" = gaea-c6 ]; then + module reset + elif [ "$L_MACHINE" = derecho ]; then module purge diff --git a/modulefiles/build_gaea-c6_intel.lua b/modulefiles/build_gaea-c6_intel.lua new file mode 100644 index 0000000000..3e05327349 --- /dev/null +++ b/modulefiles/build_gaea-c6_intel.lua @@ -0,0 +1,35 @@ +help([[ +This module loads libraries for building the UFS SRW App on +the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0 +]]) + +whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C6 ]===]) + +prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.29" +load(pathJoin("stack-cray-mpich", stack_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("srw_common") + +unload("darshan-runtime/3.4.4") +unload("cray-pmi/6.1.13") + +setenv("CFLAGS","-diag-disable=10441") +setenv("FFLAGS","-diag-disable=10441") + +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Platform","gaea-c6.intel") diff --git a/modulefiles/tasks/gaea-c6/get_obs.local.lua b/modulefiles/tasks/gaea-c6/get_obs.local.lua new file mode 100644 index 0000000000..c03abd8dfe --- /dev/null +++ b/modulefiles/tasks/gaea-c6/get_obs.local.lua @@ -0,0 +1 @@ +load("run_vx.local") diff --git a/modulefiles/tasks/gaea-c6/plot_allvars.local.lua b/modulefiles/tasks/gaea-c6/plot_allvars.local.lua new file mode 100644 index 0000000000..41da34ecca --- /dev/null +++ b/modulefiles/tasks/gaea-c6/plot_allvars.local.lua @@ -0,0 +1,4 @@ +unload("python") +load("conda") + +setenv("SRW_GRAPHICS_ENV", "srw_graphics") diff --git a/modulefiles/tasks/gaea-c6/python_srw.lua b/modulefiles/tasks/gaea-c6/python_srw.lua new file mode 100644 index 0000000000..f6b5b5fe98 --- /dev/null +++ b/modulefiles/tasks/gaea-c6/python_srw.lua @@ -0,0 +1,8 @@ +load("darshan-runtime/3.4.4") +unload("python") +load("conda") + +setenv("SRW_ENV", "srw_app") +setenv("LD_PRELOAD", "/usr/lib64/libstdc++.so.6") +setenv("FI_VERBS_PREFER_XRC", "0") + diff --git a/modulefiles/tasks/gaea-c6/run_vx.local.lua b/modulefiles/tasks/gaea-c6/run_vx.local.lua new file mode 100644 index 0000000000..be96db4bab --- /dev/null +++ b/modulefiles/tasks/gaea-c6/run_vx.local.lua @@ -0,0 +1,26 @@ +--[[ +Compiler-specific modules are used for met and metplus libraries +--]] +local met_ver = (os.getenv("met_ver") or "11.1.0") +local metplus_ver = (os.getenv("metplus_ver") or "5.1.0") +if (mode() == "load") then + load(pathJoin("met", met_ver)) + load(pathJoin("metplus",metplus_ver)) +end +local base_met = os.getenv("met_ROOT") or os.getenv("MET_ROOT") +local base_metplus = os.getenv("metplus_ROOT") or os.getenv("METPLUS_ROOT") + +setenv("MET_INSTALL_DIR", base_met) +setenv("MET_BIN_EXEC", pathJoin(base_met,"bin")) +setenv("MET_BASE", pathJoin(base_met,"share/met")) +setenv("MET_VERSION", met_ver) +setenv("METPLUS_VERSION", metplus_ver) +setenv("METPLUS_ROOT", base_metplus) +setenv("METPLUS_PATH", base_metplus) + +if (mode() == "unload") then + unload(pathJoin("met", met_ver)) + unload(pathJoin("metplus",metplus_ver)) +end +load("conda") +setenv("SRW_ENV", "srw_app") diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index 57cdfbb1cc..be96db4bab 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -24,4 +24,3 @@ if (mode() == "unload") then end load("conda") setenv("SRW_ENV", "srw_app") -setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") diff --git a/modulefiles/wflow_gaea-c6.lua b/modulefiles/wflow_gaea-c6.lua new file mode 100644 index 0000000000..ac915d0ee1 --- /dev/null +++ b/modulefiles/wflow_gaea-c6.lua @@ -0,0 +1,19 @@ +help([[ +This module loads python environement for running the UFS SRW App on +the NOAA RDHPC machine Gaea C6 +]]) + +whatis([===[Loads libraries needed for running the UFS SRW App on gaea c6 ]===]) + +unload("python") +prepend_path("MODULEPATH","/ncrc/proj/epic/rocoto/modulefiles/") +load("rocoto") +load("conda") + +pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.2.0/") + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate srw_app +]===]) +end diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index c2207ff57b..ad74ae1d14 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -3,14 +3,14 @@ This module loads python environement for running the UFS SRW App on the NOAA RDHPC machine Gaea C5 ]]) -whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) +whatis([===[Loads libraries needed for running the UFS SRW App on gaea c5 ]===]) unload("python") prepend_path("MODULEPATH","/ncrc/proj/epic/rocoto/modulefiles/") load("rocoto") load("conda") -pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.1.0/") +pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.2.0/") if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: diff --git a/tests/WE2E/machine_suites/comprehensive.gaea-c6 b/tests/WE2E/machine_suites/comprehensive.gaea-c6 new file mode 120000 index 0000000000..18a56d9739 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.gaea-c6 @@ -0,0 +1 @@ +comprehensive.orion \ No newline at end of file diff --git a/tests/WE2E/machine_suites/coverage.gaea-c6 b/tests/WE2E/machine_suites/coverage.gaea-c6 new file mode 100644 index 0000000000..970fdf4086 --- /dev/null +++ b/tests/WE2E/machine_suites/coverage.gaea-c6 @@ -0,0 +1,11 @@ +community +custom_ESGgrid_NewZealand_3km +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot +2020_CAPE +2020_easter_storm diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 8fa0977af7..44a229299a 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -45,7 +45,7 @@ function usage { } -machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion wcoss2 gaea gaea-c6 odin singularity macos noaacloud ) if [ "$1" = "-h" ] ; then usage ; fi [[ $# -le 2 ]] && usage diff --git a/tests/build.sh b/tests/build.sh index f230354a61..c592d7ea10 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea gaea-c6 odin singularity macos noaacloud ) [[ $# -gt 4 ]] && usage diff --git a/ush/machine/gaea-c6.yaml b/ush/machine/gaea-c6.yaml new file mode 100644 index 0000000000..788d21f3de --- /dev/null +++ b/ush/machine/gaea-c6.yaml @@ -0,0 +1,56 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 128 + SCHED: slurm + WE2E_TEST_DATA: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop + TEST_CCPA_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_NOHRSC_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/nohrsc/proc + DOMAIN_PREGEN_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/FV3LAM_pregen + QUEUE_DEFAULT: normal + QUEUE_FCST: normal + QUEUE_HPSS: normal + REMOVE_MEMORY: True + PARTITION_HPSS: eslogin_c6 + RUN_CMD_FCST: srun --export=ALL + RUN_CMD_POST: srun --export=ALL -n $nprocs + RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun --export=ALL -n $nprocs + SCHED_NATIVE_CMD: --clusters=c6 --partition=batch --export=NONE + SCHED_NATIVE_CMD_HPSS: --clusters=es --partition=eslogin_c6 --export=NONE + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data + TEST_PREGEN_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/FV3LAM_pregen + TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir + TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' + FIXaer: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_aer + FIXgsi: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_gsi + FIXgsm: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_am + FIXlut: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_lut + FIXorg: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_orog + FIXsfc: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_sfc_climo + FIXshp: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/NaturalEarth + EXTRN_MDL_DATA_STORES: aws +data: + ics_lbcs: + FV3GFS: + nemsio: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} + grib2: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + netcdf: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh} + RAP: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + HRRR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} + RAP: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + GSMGFS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' + nnodes: + nodesize: + ppn: diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index a8a568605e..a2899043a8 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR: ["nco", "community"] valid_vals_VERBOSE: [True, False] valid_vals_DEBUG: [True, False] -valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA", "GAEA-C6"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model"] valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"] diff --git a/ush/wrappers/job_cards/sbatch/get_ics.sbatch b/ush/wrappers/job_cards/sbatch/get_ics.sbatch index 17b6210eae..66965c504e 100644 --- a/ush/wrappers/job_cards/sbatch/get_ics.sbatch +++ b/ush/wrappers/job_cards/sbatch/get_ics.sbatch @@ -28,3 +28,7 @@ $USHdir/load_modules_run_task.sh "get_extrn_ics" $JOBSdir/JREGIONAL_GET_EXTRN_MD # Gaea differences: ##SBATCH --qos=normal ##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE + +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=es --partition=eslogin_c6 --export=NONE diff --git a/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch b/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch index 46a4aad45e..90dd6a33a0 100644 --- a/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch +++ b/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch @@ -28,3 +28,7 @@ $USHdir/load_modules_run_task.sh "get_extrn_lbcs" $JOBSdir/JREGIONAL_GET_EXTRN_M # Gaea differences: ##SBATCH --qos=normal ##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE + +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=es --partition=eslogin_c6 --export=NONE diff --git a/ush/wrappers/job_cards/sbatch/make_grid.sbatch b/ush/wrappers/job_cards/sbatch/make_grid.sbatch index 4b7dbd218c..0f9d77f6b6 100644 --- a/ush/wrappers/job_cards/sbatch/make_grid.sbatch +++ b/ush/wrappers/job_cards/sbatch/make_grid.sbatch @@ -22,6 +22,11 @@ $USHdir/load_modules_run_task.sh "make_grid" $JOBSdir/JREGIONAL_MAKE_GRID ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='24' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='24' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet #export nprocs='24' diff --git a/ush/wrappers/job_cards/sbatch/make_ics.sbatch b/ush/wrappers/job_cards/sbatch/make_ics.sbatch index 729240bdbf..a596667f52 100644 --- a/ush/wrappers/job_cards/sbatch/make_ics.sbatch +++ b/ush/wrappers/job_cards/sbatch/make_ics.sbatch @@ -28,6 +28,11 @@ $USHdir/load_modules_run_task.sh "make_ics" $JOBSdir/JREGIONAL_MAKE_ICS ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='48' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='48' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet #export nprocs='48' diff --git a/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch b/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch index d4db098b28..66b6b76ca9 100644 --- a/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch +++ b/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch @@ -30,6 +30,11 @@ $USHdir/load_modules_run_task.sh "make_lbcs" $JOBSdir/JREGIONAL_MAKE_LBCS ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='48' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='48' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet #export nprocs='48' diff --git a/ush/wrappers/job_cards/sbatch/make_orog.sbatch b/ush/wrappers/job_cards/sbatch/make_orog.sbatch index b0c8d21e54..2e86420e1d 100644 --- a/ush/wrappers/job_cards/sbatch/make_orog.sbatch +++ b/ush/wrappers/job_cards/sbatch/make_orog.sbatch @@ -22,6 +22,11 @@ $USHdir/load_modules_run_task.sh "make_orog" $JOBSdir/JREGIONAL_MAKE_OROG ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='24' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='24' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet #export nprocs='24' diff --git a/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch b/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch index 52769cb033..f8d655e64c 100644 --- a/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch +++ b/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch @@ -22,6 +22,11 @@ $USHdir/load_modules_run_task.sh "make_sfc_climo" $JOBSdir/JREGIONAL_MAKE_SFC_CL ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='48' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='48' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet #export nprocs='48' diff --git a/ush/wrappers/job_cards/sbatch/run_fcst.sbatch b/ush/wrappers/job_cards/sbatch/run_fcst.sbatch index 056fd70a28..ca10d5946b 100644 --- a/ush/wrappers/job_cards/sbatch/run_fcst.sbatch +++ b/ush/wrappers/job_cards/sbatch/run_fcst.sbatch @@ -26,6 +26,11 @@ $USHdir/load_modules_run_task.sh "run_fcst" $JOBSdir/JREGIONAL_RUN_FCST ##SBATCH --clusters=c5 --partition=batch --export=NONE #export nprocs='48' +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE +#export nprocs='48' + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet diff --git a/ush/wrappers/job_cards/sbatch/run_post.sbatch b/ush/wrappers/job_cards/sbatch/run_post.sbatch index 6af04693f3..b8dea5fa72 100644 --- a/ush/wrappers/job_cards/sbatch/run_post.sbatch +++ b/ush/wrappers/job_cards/sbatch/run_post.sbatch @@ -30,6 +30,10 @@ done ##SBATCH --qos=normal ##SBATCH --clusters=c5 --partition=batch --export=NONE +# Gaea-c6 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c6 --partition=batch --export=NONE + # Jet differences: ##SBATCH --partition=sjet,vjet,kjet,xjet