From 8bafe2972a659e616b552b2f7da415f3d353605d Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 26 Dec 2024 09:06:12 -0600 Subject: [PATCH 1/6] added sourcing of gw_setup.sh explicitly where ever it was needed for a shell call to the node --- ci/Jenkinsfile | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b7a29e15b0..67ac0fdb17 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -138,8 +138,14 @@ pipeline { } } try { - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") - gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + """) + gist_url=sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -158,7 +164,10 @@ pipeline { } } // Get a list of CI cases to run - CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + CI_CASES = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine} + """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } } @@ -179,7 +188,10 @@ pipeline { script { env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + error_output = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml + """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) error("Case ${caseName} failed to create experiment directory") @@ -194,10 +206,19 @@ pipeline { def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'") - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + """) + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + """) } catch (Exception error_experment) { - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + """) ws(CUSTOM_WORKSPACE) { def error_logs = "" def error_logs_message = "" @@ -217,9 +238,15 @@ pipeline { } } try { - gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() + gist_url = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } From 8b36793a1f4795b57a5ad75608928825148dcc57 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 26 Dec 2024 16:03:21 -0600 Subject: [PATCH 2/6] moved HOMEgfs to HOMEGFS_ as it is not used to point to a build to an experment which should be its sole purpose, also did a which on gh incase it is not in ~/bin --- ci/scripts/utils/launch_java_agent.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index eb78d3b1ef..6761639f1f 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) ######################################################################### # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEGFS_}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaea) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; @@ -84,10 +84,10 @@ esac LOG=lanuched_agent-$(date +%Y%m%d%M).log rm -f "${LOG}" -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" +source "${HOMEGFS_}/ush/module-setup.sh" +module use "${HOMEGFS_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" -source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" +source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" if [[ ! -d "${JAVA_HOME}" ]]; then @@ -102,7 +102,7 @@ JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version -export GH="${HOME}/bin/gh" +export GH=$(which gh || echo "${HOME}/bin/gh") [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" ${GH} --version From c9afe538ded41e432ccc12d8e2cfaf4f5f4e1f9f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 26 Dec 2024 16:17:06 -0600 Subject: [PATCH 3/6] updated check for GitHub CLI --- ci/scripts/utils/launch_java_agent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 6761639f1f..e307323036 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -102,8 +102,8 @@ JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version -export GH=$(which gh || echo "${HOME}/bin/gh") -[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" +GH=$(command -v gh || echo "~/bin/gh") +[[ -f "${GH}" ]] || echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ${GH} --version check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true From da4dd72e6f64b85a0c5c94ee6916f013990658d0 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 26 Dec 2024 16:18:56 -0600 Subject: [PATCH 4/6] Shell norms does not like ~ and used HOME for gh instead --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index e307323036..cade5f55a7 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -102,7 +102,7 @@ JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version -GH=$(command -v gh || echo "~/bin/gh") +GH=$(command -v gh || echo "${HOME}/bin/gh") [[ -f "${GH}" ]] || echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ${GH} --version From 623cf18e4c65d56b5d974b8c0afe592b37053309 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 26 Dec 2024 16:21:32 -0600 Subject: [PATCH 5/6] oops also need to export GH from launch for shell commands that use it --- ci/scripts/utils/launch_java_agent.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index cade5f55a7..bfc5602947 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -105,6 +105,7 @@ ${JAVA} -version GH=$(command -v gh || echo "${HOME}/bin/gh") [[ -f "${GH}" ]] || echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ${GH} --version +export GH check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then From 6c686b71a6dfcd5c9ec500af0fa869e4c3856423 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 2 Jan 2025 08:36:52 -0500 Subject: [PATCH 6/6] Update ci/scripts/utils/launch_java_agent.sh Yes, shellnorms spotted this too. Co-authored-by: Rahul Mahajan --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index bfc5602947..ad79a75cbd 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -103,7 +103,7 @@ echo "JAVA VERSION: " ${JAVA} -version GH=$(command -v gh || echo "${HOME}/bin/gh") -[[ -f "${GH}" ]] || echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 +[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ) ${GH} --version export GH