add default log rotation for Slurm daemon logs

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/config.py

@@ -32,6 +32,8 @@ class Config:
     # Set true if you want to use FSx OpenZFS in addition to FSxL. 
     enable_fsx_openzfs = False
 
+    # Set false if you want to disable log rotation of Slurm daemon logs
+    enable_slurm_log_rotation = True
 
     s3_bucket = "" # required when enable_mount_s3 = True, replace with your actual data bucket name in quotes, ie. "my-dataset-bucket"
 

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/lifecycle_script.py

@@ -271,6 +271,9 @@ def main(args):
         if Config.enable_mount_s3:
             ExecuteBashScript("./utils/mount-s3.sh").run(Config.s3_bucket)
 
+        if Config.enable_slurm_log_rotation:
+            ExecuteBashScript("./utils/enable_slurm_log_rotation.sh").run()
+
     print("[INFO]: Success: All provisioning scripts completed")
 
 

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/enable_slurm_log_rotation.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+LOGROTATE_CONF_FILEPATH="/etc/logrotate.d/sagemaker-hyperpod-slurm"
+
+echo "[$(hostname)] Adding Slurm log rotation configuration to ${LOGROTATE_CONF_FILEPATH}"
+
+cat <<EOF >>${LOGROTATE_CONF_FILEPATH}
+"/var/log/slurm/*.log" {
+    rotate 2
+    size 50M
+    copytruncate
+    nocompress
+
+    missingok
+    nodelaycompress
+    nomail
+    notifempty
+    noolddir
+    sharedscripts
+    postrotate
+        pkill -x --signal SIGUSR2 slurmctld
+        pkill -x --signal SIGUSR2 slurmd
+        pkill -x --signal SIGUSR2 slurmdbd
+        exit 0
+    endscript
+}
+EOF