diff --git a/README.md b/README.md index f0fe81c8..15cf819a 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ sleep 1000 - Instance Type: You can specify the OCI instance type that you’d like to run on as a constraint. This will make sure that you run on the right shape and also generate the right cluster. Instance types are defined in the `/opt/oci-hpc/conf/queues.conf` file in yml format. Leave all of the field in there even if they are not used. You can define multiple queues and multiple instance type in each queue. If you do not select an instance type when creating your job, it will use the default one. +- cpu-bind: On Ubuntu 22.04, we are switching to Cgroup v2 and we did notice that when hyperthreading is turned off. The default cpu-bind may give some issues. If you get an error like `error: task_g_set_affinity: Invalid argument`, you can try running your job with --cpu-bind=none or --cpu-bind=sockets ## Clusters folders: ``` /opt/oci-hpc/autoscaling/clusters/clustername diff --git a/autoscaling/crontab/autoscale_slurm.sh b/autoscaling/crontab/autoscale_slurm.sh index bc5ce204..9882b44c 100755 --- a/autoscaling/crontab/autoscale_slurm.sh +++ b/autoscaling/crontab/autoscale_slurm.sh @@ -41,7 +41,7 @@ def getTopology(clusterName): # Get the list of Jobs in all states def getJobs(): # changing the position of Dependency as it is giving blank instead of null. to handle that, putting it at the end. - out = subprocess.Popen(['squeue','-O','STATE,JOBID,FEATURE:100,NUMNODES,Partition,UserName,Dependency'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + out = subprocess.Popen(['squeue','-r','-O','STATE,JOBID,FEATURE:100,NUMNODES,Partition,UserName,Dependency'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout,stderr = out.communicate() return stdout.split("\n")[1:] @@ -433,4 +433,4 @@ try: except Exception: traceback.print_exc() -os.remove(lockfile) \ No newline at end of file +os.remove(lockfile) diff --git a/autoscaling/tf_init/bastion_update.tf b/autoscaling/tf_init/bastion_update.tf index 3205d2d2..d4154c2e 100755 --- a/autoscaling/tf_init/bastion_update.tf +++ b/autoscaling/tf_init/bastion_update.tf @@ -54,8 +54,10 @@ resource "local_file" "inventory" { spack = var.spack, ldap = var.ldap, bastion_block = var.bastion_block, + login_block = var.login_block, scratch_nfs_type = local.scratch_nfs_type, bastion_mount_ip = var.bastion_mount_ip, + login_mount_ip = var.login_mount_ip, cluster_mount_ip = local.mount_ip, cluster_name = local.cluster_name, shape = var.cluster_network ? var.cluster_network_shape : var.instance_pool_shape, @@ -72,7 +74,8 @@ resource "local_file" "inventory" { bastion_username = var.bastion_username, compute_username = var.compute_username, pam = var.pam, - sacct_limits = var.sacct_limits + sacct_limits = var.sacct_limits, + use_compute_agent=var.use_compute_agent }) filename = "${local.bastion_path}/inventory" } diff --git a/autoscaling/tf_init/cluster-network-configuration.tf b/autoscaling/tf_init/cluster-network-configuration.tf index e4f75bd7..9b1d0972 100755 --- a/autoscaling/tf_init/cluster-network-configuration.tf +++ b/autoscaling/tf_init/cluster-network-configuration.tf @@ -18,8 +18,32 @@ resource "oci_core_instance_configuration" "cluster-network-instance_configurati user_data = base64encode(data.template_file.config.rendered) } agent_config { - is_management_disabled = true + + are_all_plugins_disabled = false + is_management_disabled = true + is_monitoring_disabled = false + + plugins_config { + desired_state = "DISABLED" + name = "OS Management Service Agent" + } + dynamic plugins_config { + + for_each = var.use_compute_agent ? ["ENABLED"] : ["DISABLED"] + content { + name = "Compute HPC RDMA Authentication" + desired_state = plugins_config.value + } + } + dynamic plugins_config { + for_each = var.use_compute_agent ? ["ENABLED"] : ["DISABLED"] + content { + name = "Compute HPC RDMA Auto-Configuration" + desired_state = plugins_config.value + } + } + } shape = var.cluster_network_shape source_details { source_type = "image" diff --git a/autoscaling/tf_init/inventory.tpl b/autoscaling/tf_init/inventory.tpl index 261c1e17..146d5cce 100755 --- a/autoscaling/tf_init/inventory.tpl +++ b/autoscaling/tf_init/inventory.tpl @@ -41,8 +41,10 @@ pyxis = ${pyxis} enroot = ${enroot} spack = ${spack} bastion_block = ${bastion_block} +login_block = ${login_block} scratch_nfs_type = ${scratch_nfs_type} bastion_mount_ip = ${bastion_mount_ip} +login_mount_ip = ${login_mount_ip} cluster_mount_ip = ${cluster_mount_ip} autoscaling = true force = no @@ -68,4 +70,5 @@ latency_check=${latency_check} compute_username=${compute_username} bastion_username=${bastion_username} pam = ${pam} -sacct_limits=${sacct_limits} \ No newline at end of file +sacct_limits=${sacct_limits} +use_compute_agent=${use_compute_agent} \ No newline at end of file diff --git a/autoscaling/tf_init/marketplace.tf b/autoscaling/tf_init/marketplace.tf index 721a1c89..69d46f93 100755 --- a/autoscaling/tf_init/marketplace.tf +++ b/autoscaling/tf_init/marketplace.tf @@ -1,6 +1,6 @@ locals { - mp_listing_id = var.use_marketplace_image ? var.use_old_marketplace_image ? var.old_marketplace_listing_id : substr(var.marketplace_listing,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" - mp_version_id = var.use_old_marketplace_image ? var.marketplace_version_id[split(".", var.marketplace_listing)[0]] : var.marketplace_version_id[var.marketplace_listing] + mp_listing_id = var.use_marketplace_image ? substr(var.marketplace_listing,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" + mp_version_id = var.marketplace_version_id[var.marketplace_listing] } /* diff --git a/bastion.tf b/bastion.tf index fbfed806..c8df39cb 100644 --- a/bastion.tf +++ b/bastion.tf @@ -15,6 +15,7 @@ resource "oci_core_volume_attachment" "bastion_volume_attachment" { instance_id = oci_core_instance.bastion.id display_name = "${local.cluster_name}-bastion-volume-attachment" device = "/dev/oracleoci/oraclevdb" + is_shareable = true } resource "oci_core_volume_backup_policy" "bastion_boot_volume_backup_policy" { @@ -200,7 +201,7 @@ resource "null_resource" "bastion" { } provisioner "file" { - content = tls_private_key.ssh.private_key_pem + content = tls_private_key.ssh.private_key_openssh destination = "/home/${var.bastion_username}/.ssh/cluster.key" connection { host = local.host @@ -209,6 +210,17 @@ resource "null_resource" "bastion" { private_key = tls_private_key.ssh.private_key_pem } } + + provisioner "file" { + content = tls_private_key.ssh.public_key_openssh + destination = "/home/${var.bastion_username}/.ssh/id_rsa.pub" + connection { + host = local.host + type = "ssh" + user = var.bastion_username + private_key = tls_private_key.ssh.private_key_pem + } + } } resource "null_resource" "cluster" { depends_on = [null_resource.bastion, null_resource.backup, oci_core_compute_cluster.compute_cluster, oci_core_cluster_network.cluster_network, oci_core_instance.bastion, oci_core_volume_attachment.bastion_volume_attachment ] @@ -246,14 +258,17 @@ resource "null_resource" "cluster" { log_vol = var.log_vol, redundancy = var.redundancy, cluster_network = var.cluster_network, + use_compute_agent = var.use_compute_agent, slurm = var.slurm, rack_aware = var.rack_aware, slurm_nfs_path = var.slurm_nfs ? var.nfs_source_path : var.cluster_nfs_path spack = var.spack, ldap = var.ldap, bastion_block = var.bastion_block, + login_block = var.login_block, scratch_nfs_type = local.scratch_nfs_type, bastion_mount_ip = local.bastion_mount_ip, + login_mount_ip = local.login_mount_ip, cluster_mount_ip = local.mount_ip, autoscaling = var.autoscaling, cluster_name = local.cluster_name, @@ -324,11 +339,11 @@ resource "null_resource" "cluster" { provisioner "file" { content = templatefile("${path.module}/queues.conf", { cluster_network = var.cluster_network, + use_compute_agent = var.use_compute_agent, compute_cluster = var.compute_cluster, - marketplace_listing = var.use_old_marketplace_image ? var.old_marketplace_listing : var.marketplace_listing, + marketplace_listing = var.marketplace_listing, image = local.image_ocid, use_marketplace_image = var.use_marketplace_image, - use_old_marketplace_image = var.use_old_marketplace_image, boot_volume_size = var.boot_volume_size, shape = var.cluster_network ? var.cluster_network_shape : var.instance_pool_shape, region = var.region, @@ -376,8 +391,10 @@ resource "null_resource" "cluster" { spack = var.spack, ldap = var.ldap, bastion_block = var.bastion_block, + login_block = var.login_block, scratch_nfs_type = local.scratch_nfs_type, bastion_mount_ip = local.bastion_mount_ip, + login_mount_ip = local.login_mount_ip, cluster_mount_ip = local.mount_ip, scratch_nfs_type_cluster = var.scratch_nfs_type_cluster, scratch_nfs_type_pool = var.scratch_nfs_type_pool, @@ -390,8 +407,6 @@ resource "null_resource" "cluster" { ssh_cidr = var.ssh_cidr, use_cluster_nfs = var.use_cluster_nfs, cluster_nfs_path = var.cluster_nfs_path, - bastion_block = var.bastion_block, - bastion_mount_ip = local.bastion_mount_ip, home_nfs = var.home_nfs, create_fss = var.create_fss, home_fss = var.home_fss, @@ -417,7 +432,8 @@ resource "null_resource" "cluster" { bastion_username = var.bastion_username, compute_username = var.compute_username, pam = var.pam, - sacct_limits = var.sacct_limits + sacct_limits = var.sacct_limits, + use_compute_agent = var.use_compute_agent }) destination = "/opt/oci-hpc/conf/variables.tf" @@ -467,6 +483,7 @@ provisioner "file" { "chmod a+x /opt/oci-hpc/bin/*.sh", "timeout --foreground 60m /opt/oci-hpc/bin/bastion.sh", "chmod 755 /opt/oci-hpc/autoscaling/crontab/*.sh", + "chmod 755 /opt/oci-hpc/samples/*.sh", "chmod 600 /opt/oci-hpc/autoscaling/credentials/key.pem", "echo ${var.configure} > /tmp/configure.conf", "timeout 2h /opt/oci-hpc/bin/configure.sh | tee /opt/oci-hpc/logs/initial_configure.log", @@ -487,7 +504,9 @@ data "oci_objectstorage_namespace" "compartment_namespace" { } locals { - rdma_nic_metric_bucket_name = "RDMA_NIC_metrics" + current_timestamp = timestamp() + current_timestamp_formatted = formatdate("YYYYMMDDhhmmss", local.current_timestamp) + rdma_nic_metric_bucket_name = format("%s_%s","RDMA_NIC_metrics",local.current_timestamp_formatted) par_path = ".." } /* @@ -495,14 +514,9 @@ saving the PAR into file: ../PAR_file_for_metrics. this PAR is used by the scripts to upload NIC metrics to object storage (i.e. script: upload_rdma_nic_metrics.sh) */ -data "oci_objectstorage_bucket" "RDMA_NIC_Metrics_bucket_check" { - name = local.rdma_nic_metric_bucket_name - namespace = data.oci_objectstorage_namespace.compartment_namespace.namespace -} - resource "oci_objectstorage_bucket" "RDMA_NIC_metrics_bucket" { - count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0 + count = (var.bastion_object_storage_par) ? 1 : 0 compartment_id = var.targetCompartment name = local.rdma_nic_metric_bucket_name namespace = data.oci_objectstorage_namespace.compartment_namespace.namespace @@ -510,7 +524,7 @@ resource "oci_objectstorage_bucket" "RDMA_NIC_metrics_bucket" { } resource "oci_objectstorage_preauthrequest" "RDMA_NIC_metrics_par" { - count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0 + count = (var.bastion_object_storage_par) ? 1 : 0 depends_on = [oci_objectstorage_bucket.RDMA_NIC_metrics_bucket] access_type = "AnyObjectWrite" bucket = local.rdma_nic_metric_bucket_name @@ -522,12 +536,12 @@ resource "oci_objectstorage_preauthrequest" "RDMA_NIC_metrics_par" { output "RDMA_NIC_metrics_url" { depends_on = [oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par] - value = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}" : "" + value = (var.bastion_object_storage_par) ? "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}" : "" } resource "local_file" "PAR" { - count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0 + count = (var.bastion_object_storage_par) ? 1 : 0 depends_on = [oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par] content = "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}" filename = "${local.par_path}/PAR_file_for_metrics" diff --git a/bin/bastion.sh b/bin/bastion.sh index 0cfc7d31..501cc559 100644 --- a/bin/bastion.sh +++ b/bin/bastion.sh @@ -43,15 +43,15 @@ if [ $ID == "ol" ] || [ $ID == "centos" ] ; then sudo mkdir /etc/ansible sudo ln -s /usr/local/bin/ansible-playbook /bin/ansible-playbook sudo ln -s /usr/local/bin/ansible /bin/ansible - sudo python3 -m pip install -U pip - sudo python3 -m pip install netaddr --upgrade - sudo python3 -m pip install setuptools_rust --upgrade - sudo python3 -m pip install requests --upgrade - sudo python3 -m pip install urllib3 --upgrade fi sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo sudo yum install -y terraform - sudo python3 -m pip install oci-cli --upgrade + sudo python3 -m pip install -U pip + sudo python3 -m pip install netaddr --upgrade + sudo python3 -m pip install setuptools_rust --upgrade + sudo python3 -m pip install requests --upgrade + sudo python3 -m pip install urllib3 --upgrade + sudo python3 -m pip install oci-cli --upgrade elif [ $ID == "debian" ] || [ $ID == "ubuntu" ] ; then @@ -91,24 +91,41 @@ elif [ $ID == "debian" ] || [ $ID == "ubuntu" ] ; then sudo apt -y --fix-broken install fix_apt - + + sudo add-apt-repository --yes --update ppa:ansible/ansible sudo apt-get -y install ansible output=$? if [ $output -ne 0 ] then fix_apt + sleep 60s sudo apt-get -y install ansible fi fix_apt - sudo apt-get -y install python python-netaddr python3 python3-pip + + if [ $ID == "ubuntu" ] && [ $VERSION_ID == "22.04" ] ; then + sudo sed -i 's/#$nrconf{restart} = '"'"'i'"'"';/$nrconf{restart} = '"'"'a'"'"';/g' /etc/needrestart/needrestart.conf + sudo apt-get -y install python3 python3-netaddr python3-pip + sudo ln -s /usr/bin/python3 /usr/bin/python + else + sudo apt-get -y install python python-netaddr python3 python3-pip + fi output=$? if [ $output -ne 0 ] then fix_apt - sudo apt-get -y install python python-netaddr python3 python3-pip + if [ $ID == "ubuntu" ] && [ $VERSION_ID == "22.04" ] ; then + sudo apt-get -y install python3 python3-netaddr python3-pip + + else + sudo apt-get -y install python python-netaddr python3 python3-pip + fi fi fix_apt - + sudo python3 -m pip install -U pip + sudo python3 -m pip install netaddr --upgrade + sudo python3 -m pip install requests --upgrade + sudo python3 -m pip install urllib3 --upgrade pip install pip --upgrade pip install pyopenssl --upgrade @@ -153,11 +170,22 @@ ansible-galaxy collection install community.crypto --force > /dev/null threads=$(nproc) forks=$(($threads * 8)) -sudo sed -i "s/^#forks.*/forks = ${forks}/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#fact_caching=.*/fact_caching=jsonfile/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#fact_caching_connection.*/fact_caching_connection=\/tmp\/ansible/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#bin_ansible_callbacks.*/bin_ansible_callbacks=True/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#stdout_callback.*/stdout_callback=yaml/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#retries.*/retries=5/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#connect_timeout.*/connect_timeout=300/" /etc/ansible/ansible.cfg -sudo sed -i "s/^#command_timeout.*/command_timeout=120/" /etc/ansible/ansible.cfg \ No newline at end of file +if [ ! -d /etc/ansible ] ; then + sudo mkdir /etc/ansible + if [ $ID == "ubuntu" ] ; then + sudo chown ubuntu:ubuntu /etc/ansible + else + sudo chown opc:opc /etc/ansible + fi +fi + +ansible-config init --disabled -t all | sudo tee /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)forks.*/forks = ${forks}/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)fact_caching=.*/fact_caching=jsonfile/" /etc/ansible/ansible.cfg +sudo sed -i "0,/^\(#\|;\)fact_caching_connection.*/s//fact_caching_connection=\/tmp\/ansible/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)bin_ansible_callbacks.*/bin_ansible_callbacks=True/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)stdout_callback.*/stdout_callback=yaml/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)retries.*/retries=5/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)connect_timeout.*/connect_timeout=300/" /etc/ansible/ansible.cfg +sudo sed -i "s/^\(#\|;\)command_timeout.*/command_timeout=120/" /etc/ansible/ansible.cfg + diff --git a/bin/create_cluster.sh b/bin/create_cluster.sh index 2529a124..df2bb3d5 100755 --- a/bin/create_cluster.sh +++ b/bin/create_cluster.sh @@ -42,7 +42,6 @@ targetCompartment=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_typ ADNames=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.ad " $queues_conf` boot_volume_size=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.boot_volume_size " $queues_conf` use_marketplace_image=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.use_marketplace_image " $queues_conf` -use_old_marketplace_image=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.use_old_marketplace_image " $queues_conf` image=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.image " $queues_conf` instance_pool_ocpus=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.instance_pool_ocpus " $queues_conf` instance_pool_memory=`yq eval ".queues.[] | select(.name == \"$4\") | .instance_types.[] | select(.name == \"$3\") |.instance_pool_memory " $queues_conf` @@ -68,7 +67,7 @@ do echo $1 $3 $4 >> currently_building echo $3 $4 > cluster_options - sed "s~##NODES##~$1~g;s~##NAME##~$2~g;s~##SHAPE##~$shape~g;s~##CN##~$cluster_network~g;s~##QUEUE##~${4}~g;s~##COMP##~${targetCompartment}~g;s~##AD##~${ADName}~g;s~##BOOT##~${boot_volume_size}~g;s~##USEMP##~${use_marketplace_image}~g;s~##USEOLDMP##~${use_old_marketplace_image}~g;s~##IMAGE##~${image}~g;s~##OCPU##~${instance_pool_ocpus}~g;s~##MEM##~${instance_pool_memory}~g;s~##CUSTOM_MEM##~${instance_pool_custom_memory}~g;s~##MP_LIST##~${marketplace_listing}~g;s~##HT##~${hyperthreading}~g;s~##INST_TYPE##~$3~g;s~##TAGS##~$tags~g;s~##REGION##~${region}~g;s~##PRIVATE_SUBNET_ID##~${private_subnet_id}~g;s~##PRIVATE_SUBNET##~${private_subnet}~g;s~##CC##~$compute_cluster~g" $conf_folder/variables.tf > variables.tf + sed "s~##NODES##~$1~g;s~##NAME##~$2~g;s~##SHAPE##~$shape~g;s~##CN##~$cluster_network~g;s~##QUEUE##~${4}~g;s~##COMP##~${targetCompartment}~g;s~##AD##~${ADName}~g;s~##BOOT##~${boot_volume_size}~g;s~##USEMP##~${use_marketplace_image}~g;s~##IMAGE##~${image}~g;s~##OCPU##~${instance_pool_ocpus}~g;s~##MEM##~${instance_pool_memory}~g;s~##CUSTOM_MEM##~${instance_pool_custom_memory}~g;s~##MP_LIST##~${marketplace_listing}~g;s~##HT##~${hyperthreading}~g;s~##INST_TYPE##~$3~g;s~##TAGS##~$tags~g;s~##REGION##~${region}~g;s~##PRIVATE_SUBNET_ID##~${private_subnet_id}~g;s~##PRIVATE_SUBNET##~${private_subnet}~g;s~##CC##~$compute_cluster~g" $conf_folder/variables.tf > variables.tf echo "Started to build $2" start=`date -u +%s` diff --git a/bin/resize.py b/bin/resize.py index 48008763..5faf6273 100644 --- a/bin/resize.py +++ b/bin/resize.py @@ -461,29 +461,33 @@ def get_summary(comp_ocid,cluster_name): elif cn_summary_tmp.lifecycle_state == "SCALING": scaling_clusters = scaling_clusters + 1 if running_clusters == 0: - cn_summaries = computeClient.list_compute_clusters(comp_ocid,display_name=cluster_name).data.items + try: + cn_summaries = computeClient.list_compute_clusters(comp_ocid,display_name=cluster_name).data.items + except: + print("The list_compute_clusters call returned an error, considering no Compute CLusters are present") + cn_summaries = [] if len(cn_summaries) > 0: CN = "CC" for cn_summary_tmp in cn_summaries: if cn_summary_tmp.lifecycle_state == "ACTIVE" and cn_summary_tmp.display_name == cluster_name : cn_summary = cn_summary_tmp running_clusters = running_clusters + 1 + if running_clusters == 0: + cn_summaries = computeManagementClient.list_instance_pools(comp_ocid,display_name=cluster_name).data + if len(cn_summaries) > 0: + CN = "IP" + for cn_summary_tmp in cn_summaries: + if cn_summary_tmp.lifecycle_state == "RUNNING": + cn_summary = cn_summary_tmp + running_clusters = running_clusters + 1 + elif cn_summary_tmp.lifecycle_state == "SCALING": + scaling_clusters = scaling_clusters + 1 if running_clusters == 0: - cn_summaries = computeManagementClient.list_instance_pools(comp_ocid,display_name=cluster_name).data - if len(cn_summaries) > 0: - CN = "IP" - for cn_summary_tmp in cn_summaries: - if cn_summary_tmp.lifecycle_state == "RUNNING": - cn_summary = cn_summary_tmp - running_clusters = running_clusters + 1 - elif cn_summary_tmp.lifecycle_state == "SCALING": - scaling_clusters = scaling_clusters + 1 - if running_clusters == 0: - if scaling_clusters: - print("No running cluster was found but there is a cluster in SCALING mode, try rerunning in a moment") - else: - print("The cluster was not found") - return None,None,True + if scaling_clusters: + print("No running cluster was found but there is a cluster in SCALING mode, try rerunning in a moment") + else: + print("The cluster was not found") + return None,None,True if running_clusters > 1: print("There were multiple running clusters with this name, we selected the one with OCID:"+cn_summary.id) if CN == "CN": diff --git a/bin/upload_rdma_nic_metrics.sh b/bin/upload_rdma_nic_metrics.sh index 6e44671c..59486baf 100644 --- a/bin/upload_rdma_nic_metrics.sh +++ b/bin/upload_rdma_nic_metrics.sh @@ -7,6 +7,7 @@ source "${folder}/rdma_metrics_collection_config.conf" hours="$hoursAgoFromNow" interval="$metricsCollectionIntervalInMinute" par_filename="$parFileName" +cluster_name="" if [ -z "$par_filename" ] then @@ -25,16 +26,17 @@ dis_help() echo echo "Usage:" echo - echo "./upload_rdma_nic_metrics.sh -l -i " + echo "./upload_rdma_nic_metrics.sh -l -i -c " echo echo "Options:" echo "l Hours Ago From Now (optional)" - echo "n Metrics Collection Interval In Minute (optional)" + echo "i Metrics Collection Interval In Minute (optional)" + echo "c Cluster Name (optional)" echo "h Print this help." echo echo "RDMA metrics are uploaded to Object Storage using PAR" echo - echo "e.g., sh ./upload_rdma_nic_metrics.sh -l 24 -i 5 " + echo "e.g., sh ./upload_rdma_nic_metrics.sh -l 24 -i 5 -c clusterName1" echo echo "Supported releases: 2.10.3+" echo @@ -43,11 +45,12 @@ dis_help() #Do this if number of arguments passed is greater than 0 if [ "$#" -gt "0" ] then - while getopts "l:i:h" option + while getopts "l:i:c:h" option do case $option in l) hours=${OPTARG};; i) interval=${OPTARG};; + c) cluster_name=${OPTARG};; h) dis_help exit;; \?) # Invalid option @@ -78,7 +81,7 @@ then filename="infiniband_mlx5_${i}_${timestamp}" filename_csv="${filename}.csv" - filename_zip="${filename}.zip" + filename_zip="${cluster_name}_${filename}.zip" echo "Collecting RDMA HW metrics of device mlx5_${i}...." query="SELECT * FROM ${measurementnameBackup}" @@ -123,7 +126,7 @@ then filename="infiniband_${timestamp}" filename_csv="${filename}.csv" - filename_zip="${filename}.zip" + filename_zip="${cluster_name}_${filename}.zip" echo "Collecting Infiniband counter metrics...." query="SELECT * FROM ${measurementnameBackup}" diff --git a/cluster-network-configuration.tf b/cluster-network-configuration.tf index b66b1967..82a3fb60 100755 --- a/cluster-network-configuration.tf +++ b/cluster-network-configuration.tf @@ -22,8 +22,34 @@ resource "oci_core_instance_configuration" "cluster-network-instance_configurati user_data = base64encode(data.template_file.config.rendered) } agent_config { - is_management_disabled = true + + are_all_plugins_disabled = false + is_management_disabled = true + is_monitoring_disabled = false + + plugins_config { + desired_state = "DISABLED" + name = "OS Management Service Agent" + } + dynamic plugins_config { + + for_each = var.use_compute_agent ? ["ENABLED"] : ["DISABLED"] + content { + name = "Compute HPC RDMA Authentication" + desired_state = plugins_config.value + } + } + dynamic plugins_config { + for_each = var.use_compute_agent ? ["ENABLED"] : ["DISABLED"] + content { + name = "Compute HPC RDMA Auto-Configuration" + desired_state = plugins_config.value + } + } + } + + shape = var.cluster_network_shape source_details { source_type = "image" diff --git a/conf/queues.conf.example b/conf/queues.conf.example index d75dc088..fe187437 100644 --- a/conf/queues.conf.example +++ b/conf/queues.conf.example @@ -19,9 +19,8 @@ private_subnet_id: ocid1.subnet.oc1..aaaaaaaaf5greyx5b45hkh7rwm2osef7t5lofl5tag5uijjqlhw6duaaaaa #TO EDIT image: ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa #TO EDIT targetCompartment: ocid1.compartment.oc1.. #TO EDIT - boot_volume_size: 50 + boot_volume_size: 100 use_marketplace_image: true - use_old_marketplace_image: false instance_pool_ocpus: 2 instance_pool_memory: 16 instance_pool_custom_memory: false @@ -43,9 +42,8 @@ private_subnet_id: ocid1.subnet.oc1..aaaaaaaaf5greyx5b45hkh7rwm2osef7t5lofl5tag5uijjqlhw6duaaaaa #TO EDIT image: ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa #TO EDIT targetCompartment: ocid1.compartment.oc1.. #TO EDIT - boot_volume_size: 50 + boot_volume_size: 100 use_marketplace_image: true - use_old_marketplace_image: false instance_pool_ocpus: 2 instance_pool_memory: 16 instance_pool_custom_memory: false @@ -69,9 +67,8 @@ private_subnet_id: ocid1.subnet.oc1..aaaaaaaaf5greyx5b45hkh7rwm2osef7t5lofl5tag5uijjqlhw6duaaaaa #TO EDIT image: ocid1.image.oc1.eu-frankfurt-1.aaaaaaaasrw572lodukbaxlwrrq5zcqe3zldh3krkgtq3lm463lann4ksxeq #TO EDIT targetCompartment: ocid1.compartment.oc1.. #TO EDIT - boot_volume_size: 50 + boot_volume_size: 100 use_marketplace_image: false - use_old_marketplace_image: false instance_pool_ocpus: 2 instance_pool_memory: 16 instance_pool_custom_memory: false @@ -92,9 +89,8 @@ private_subnet_id: ocid1.subnet.oc1..aaaaaaaaf5greyx5b45hkh7rwm2osef7t5lofl5tag5uijjqlhw6duaaaaa #TO EDIT image: ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa #TO EDIT targetCompartment: ocid1.compartment.oc1.. #TO EDIT - boot_volume_size: 50 + boot_volume_size: 100 use_marketplace_image: false - use_old_marketplace_image: false instance_pool_ocpus: 2 instance_pool_memory: 2 instance_pool_custom_memory: true diff --git a/conf/variables.tpl b/conf/variables.tpl index 71ffd5cb..96dd18d3 100755 --- a/conf/variables.tpl +++ b/conf/variables.tpl @@ -15,7 +15,6 @@ variable "instance_type" {default = "##INST_TYPE##" } variable "node_count" { default="##NODES##" } variable "boot_volume_size" {default = "##BOOT##"} variable "use_marketplace_image" { default = "##USEMP##" } -variable "use_old_marketplace_image" { default = "##USEOLDMP##" } variable "scratch_nfs_path" { default = "${scratch_nfs_path}" } variable "use_scratch_nfs" { default = ${use_scratch_nfs} } variable "cluster_nfs_path" {default = "${cluster_nfs_path}"} @@ -35,6 +34,7 @@ variable "pyxis" { default = ${pyxis} } variable "pam" { default = ${pam} } variable "sacct_limits" { default = ${sacct_limits} } variable "enroot" { default = ${enroot} } +variable "use_compute_agent" { default = ${use_compute_agent} } variable "slurm_nfs_path" { default = "${slurm_nfs_path}" } variable "spack" { default = ${spack} } variable "instance_pool_ocpus" { default = "##OCPU##"} @@ -53,12 +53,10 @@ variable "marketplace_version_id" { "2" = "OL7.8-OFED5.0-1.0.0.0-UEK-20200826" "3" = "OL7.7-OFED-4.4-2.0.7.0-UEK-20200229" "4" = "OL7.9-OFED5.0-2.1.8.0-RHCK-20210709" - "HPC_OL7" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.6.8.1-2023.05.18-0" - "HPC_OL8" = "OracleLinux-8-RHCK-OFED-5.4-3.6.8.1-2023.05.18-0" - "HPC_OL7_old" = "OL7.9-RHCK-3.10.0-OFED-5.4-3.4.0-1" - "HPC_OL8_old" = "OracleLinux-8-RHCK-OFED-5.4-3.5.8.0-2022.11.15-0" - "GPU_old" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.4.0.0-GPU-510-2022.09.23-1" - "GPU" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.6.8.1-GPU-515-2023.05.18-0" + "HPC_OL7" = "OracleLinux-7-OCA-RHCK-OFED-5.8-3.0.7.0-2024.01.02-0" + "HPC_OL8" = "OracleLinux-8-OCA-RHCK-OFED-5.8-3.0.7.0-2024.01.02-1" + "GPU_OL7" = "OracleLinux-7-OCA-RHCK-OFED-5.8-3.0.7.0-GPU-535-2024.01.02-0" + "GPU_OL8" = "OracleLinux-8-OCA-RHCK-OFED-5.8-3.0.7.0-GPU-535-2024.01.02-1" } } @@ -101,8 +99,10 @@ variable "cluster_block_volume_performance" {default="${cluster_block_volume_per variable "ssh_cidr" {default="${ssh_cidr}"} variable "bastion_block" {default = "${bastion_block}"} +variable "login_block" {default = "${login_block}"} variable "bastion_mount_ip" {default = "${bastion_mount_ip}"} +variable "login_mount_ip" {default = "${login_mount_ip}"} variable "home_nfs" { default = ${home_nfs} } variable "home_fss" { default = ${home_fss} } variable "latency_check" { default = ${latency_check} } diff --git a/data.tf b/data.tf index 39195858..3325be77 100755 --- a/data.tf +++ b/data.tf @@ -3,8 +3,7 @@ resource "random_pet" "name" { } resource "tls_private_key" "ssh" { - algorithm = "RSA" - rsa_bits = "4096" + algorithm = "ED25519" } data "oci_core_services" "services" { diff --git a/inventory.tpl b/inventory.tpl index 735e41b3..1d1586c2 100755 --- a/inventory.tpl +++ b/inventory.tpl @@ -35,8 +35,10 @@ slurm = ${slurm} rack_aware = ${rack_aware} spack = ${spack} bastion_block = ${bastion_block} +login_block = ${login_block} scratch_nfs_type = ${scratch_nfs_type} bastion_mount_ip = ${bastion_mount_ip} +login_mount_ip = ${login_mount_ip} cluster_mount_ip = ${cluster_mount_ip} autoscaling = ${autoscaling} cluster_name = ${cluster_name} @@ -73,4 +75,5 @@ tenancy_ocid = ${tenancy_ocid} inst_prin = ${inst_prin} api_fingerprint = ${api_fingerprint} api_user_ocid = ${api_user_ocid} -sacct_limits=${sacct_limits} \ No newline at end of file +sacct_limits=${sacct_limits} +use_compute_agent=${use_compute_agent} \ No newline at end of file diff --git a/locals.tf b/locals.tf index 38bac32c..9c791ed3 100755 --- a/locals.tf +++ b/locals.tf @@ -28,9 +28,9 @@ locals { cluster_name = var.use_custom_name ? var.cluster_name : random_pet.name.id - bastion_image = var.use_standard_image ? oci_core_app_catalog_subscription.bastion_mp_image_subscription[0].listing_resource_id : local.custom_bastion_image_ocid + bastion_image = var.use_marketplace_image_bastion ? oci_core_app_catalog_subscription.bastion_mp_image_subscription[0].listing_resource_id : local.custom_bastion_image_ocid - login_image = var.login_node && ( var.use_standard_image_login || var.use_marketplace_image_login ) ? oci_core_app_catalog_subscription.login_mp_image_subscription[0].listing_resource_id : local.custom_login_image_ocid + login_image = var.login_node && var.use_marketplace_image_login ? oci_core_app_catalog_subscription.login_mp_image_subscription[0].listing_resource_id : local.custom_login_image_ocid cluster_network_image = var.use_marketplace_image ? oci_core_app_catalog_subscription.mp_image_subscription[0].listing_resource_id : local.image_ocid @@ -42,8 +42,9 @@ locals { is_login_flex_shape = length(regexall(".*VM.*.*Flex$", var.login_shape)) > 0 ? [local.login_ocpus]:[] is_instance_pool_flex_shape = length(regexall(".*VM.*.*Flex$", var.instance_pool_shape)) > 0 ? [local.instance_pool_ocpus]:[] - + bastion_mount_ip = var.bastion_block ? element(concat(oci_core_volume_attachment.bastion_volume_attachment.*.ipv4, [""]), 0) : "none" + login_mount_ip = var.login_block ? element(concat(oci_core_volume_attachment.login_volume_attachment.*.ipv4, [""]), 0) : "none" scratch_nfs_type = var.cluster_network ? var.scratch_nfs_type_cluster : var.scratch_nfs_type_pool diff --git a/marketplace.tf b/marketplace.tf index 5917390b..c434af50 100755 --- a/marketplace.tf +++ b/marketplace.tf @@ -1,11 +1,11 @@ locals { // listing_number = split(".", var.marketplace_listing)[0] - mp_listing_id = var.use_marketplace_image ? var.use_old_marketplace_image ? var.old_marketplace_listing_id : substr(var.marketplace_listing,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" - mp_bastion_listing_id = var.use_standard_image ? var.use_old_marketplace_image ? var.old_marketplace_listing_id :var.marketplace_listing_id_HPC : "" - mp_login_listing_id = var.use_marketplace_image_login ? var.use_old_marketplace_image_login ? var.old_marketplace_listing_id : substr(var.marketplace_listing_login,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" - mp_version_id = var.use_old_marketplace_image ? var.marketplace_version_id[split(".", var.old_marketplace_listing)[0]] : var.marketplace_version_id[var.marketplace_listing] - mp_bastion_version_id = var.use_old_marketplace_image ? var.marketplace_version_id[split(".", var.old_marketplace_listing)[0]] : var.marketplace_version_id["HPC_OL7"] - mp_login_version_id = var.use_old_marketplace_image_login ? var.marketplace_version_id[split(".", var.old_marketplace_listing_login)[0]] : var.marketplace_version_id[var.marketplace_listing_login] + mp_listing_id = var.use_marketplace_image ? substr(var.marketplace_listing,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" + mp_bastion_listing_id = var.use_marketplace_image_bastion ? substr(var.marketplace_listing_bastion,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" + mp_login_listing_id = var.use_marketplace_image_login ? substr(var.marketplace_listing_login,0,3) == "HPC" ? var.marketplace_listing_id_HPC : var.marketplace_listing_id_GPU : "" + mp_version_id = var.marketplace_version_id[var.marketplace_listing] + mp_bastion_version_id = var.marketplace_version_id[var.marketplace_listing_bastion] + mp_login_version_id = var.marketplace_version_id[var.marketplace_listing_login] } /* @@ -49,12 +49,12 @@ resource "oci_core_app_catalog_subscription" "mp_image_subscription" { } data "oci_core_app_catalog_listing_resource_versions" "bastion_app_catalog_listing_resource_versions" { - count = var.use_standard_image ? 1 : 0 + count = var.use_marketplace_image_bastion ? 1 : 0 listing_id = local.mp_bastion_listing_id } resource "oci_core_app_catalog_listing_resource_version_agreement" "bastion_mp_image_agreement" { - count = var.use_standard_image ? 1 : 0 + count = ( var.use_marketplace_image_bastion ) ? 1 : 0 listing_id = local.mp_bastion_listing_id listing_resource_version = local.mp_bastion_version_id @@ -62,7 +62,7 @@ resource "oci_core_app_catalog_listing_resource_version_agreement" "bastion_mp_i } resource "oci_core_app_catalog_subscription" "bastion_mp_image_subscription" { - count = var.use_standard_image ? 1 : 0 + count = ( var.use_marketplace_image_bastion ) ? 1 : 0 compartment_id = var.targetCompartment eula_link = oci_core_app_catalog_listing_resource_version_agreement.bastion_mp_image_agreement[0].eula_link listing_id = oci_core_app_catalog_listing_resource_version_agreement.bastion_mp_image_agreement[0].listing_id @@ -77,12 +77,12 @@ resource "oci_core_app_catalog_subscription" "bastion_mp_image_subscription" { } data "oci_core_app_catalog_listing_resource_versions" "login_app_catalog_listing_resource_versions" { - count = var.login_node && ( var.use_marketplace_image_login || var.use_standard_image_login ) ? 1 : 0 + count = var.login_node && var.use_marketplace_image_login ? 1 : 0 listing_id = local.mp_login_listing_id } resource "oci_core_app_catalog_listing_resource_version_agreement" "login_mp_image_agreement" { - count = var.login_node && ( var.use_marketplace_image_login || var.use_standard_image_login ) ? 1 : 0 + count = var.login_node && var.use_marketplace_image_login ? 1 : 0 listing_id = local.mp_login_listing_id listing_resource_version = local.mp_login_version_id @@ -90,7 +90,7 @@ resource "oci_core_app_catalog_listing_resource_version_agreement" "login_mp_ima } resource "oci_core_app_catalog_subscription" "login_mp_image_subscription" { - count = var.login_node && ( var.use_marketplace_image_login || var.use_standard_image_login ) ? 1 : 0 + count = var.login_node && var.use_marketplace_image_login ? 1 : 0 compartment_id = var.targetCompartment eula_link = oci_core_app_catalog_listing_resource_version_agreement.login_mp_image_agreement[0].eula_link listing_id = oci_core_app_catalog_listing_resource_version_agreement.login_mp_image_agreement[0].listing_id diff --git a/playbooks/destroy.yml b/playbooks/destroy.yml index 520b756d..46efb661 100755 --- a/playbooks/destroy.yml +++ b/playbooks/destroy.yml @@ -15,4 +15,5 @@ destroy: true initial: false roles: - - etc-hosts \ No newline at end of file + - etc-hosts + \ No newline at end of file diff --git a/playbooks/new_nodes.yml b/playbooks/new_nodes.yml index 83e57b9c..3be0cb57 100755 --- a/playbooks/new_nodes.yml +++ b/playbooks/new_nodes.yml @@ -46,12 +46,15 @@ become: true gather_facts: true tasks: + - include_role: + name: cloud-agent_update + when: cluster_network|bool and use_compute_agent|default(false)|bool - include_role: name: oci-cn-auth - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: rdma-interface - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: nvidia_peermem @@ -176,7 +179,7 @@ tasks: - include_role: name: latency_check - when: cluster_network|bool and not 'GPU' in shape + when: cluster_network|bool and latency_check|bool - hosts: all become: true diff --git a/playbooks/resize_add.yml b/playbooks/resize_add.yml index 5703a1cf..e3070b40 100755 --- a/playbooks/resize_add.yml +++ b/playbooks/resize_add.yml @@ -44,12 +44,15 @@ become: true gather_facts: true tasks: + - include_role: + name: cloud-agent_update + when: cluster_network|bool and use_compute_agent|default(false)|bool - include_role: name: oci-cn-auth - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: rdma-interface - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: nvidia_peermem @@ -174,12 +177,6 @@ - include_role: name: tuned -- hosts: compute_to_add - tasks: - - include_role: - name: latency_check - when: cluster_network|bool and not 'GPU' in shape - - hosts: all become: true tasks: diff --git a/playbooks/roles/cloud-agent_update/tasks/el.yml b/playbooks/roles/cloud-agent_update/tasks/el.yml new file mode 100644 index 00000000..60211cd3 --- /dev/null +++ b/playbooks/roles/cloud-agent_update/tasks/el.yml @@ -0,0 +1,39 @@ +--- +- name: Check oracle cloud agent version + shell: "yum info oracle-cloud-agent | grep Version | awk '{print $3}'" + register: version + when: use_compute_agent | bool + +- name: Install OCA v1.37 for OL8 + vars: + - major_version: "{{version.stdout.split('.')[1] }}" + - minor_version: "{{version.stdout.split('.')[0] }}" + - sub_version: "{{version.stdout.split('.')[2].split('-')[0] }}" + yum: + name: "https://objectstorage.us-phoenix-1.oraclecloud.com/p/aV_mSl96KIiapAeZtsyo-SUcPCSurDfWaj06f4XVVoNKIsxvqlZ65guPTnMuNawR/n/imagegen/b/agent_test/o/1.37.0/3/oracle-cloud-agent-1.37.2-10459.el8.x86_64.rpm" + state: present + disable_gpg_check: yes + when: + - ansible_os_family == 'RedHat' + - ansible_distribution_major_version == '8' + - (minor_version | int <= 1) | bool + - (major_version | int <= 37) | bool + - (sub_version | int < 2) | bool + - use_compute_agent | bool + +- name: Install OCA v1.37 for OL7 + vars: + - major_version: "{{version.stdout.split('.')[1] }}" + - minor_version: "{{version.stdout.split('.')[0] }}" + - sub_version: "{{version.stdout.split('.')[2].split('-')[0] }}" + yum: + name: "https://objectstorage.us-phoenix-1.oraclecloud.com/p/YmPlysZFl4CKrLTKN9Rj0CMPt8qiJgflvF4vXsOaaqOfcm5NMnyBJl_dlC0V0lTo/n/imagegen/b/agent_test/o/1.37.0/3/oracle-cloud-agent-1.37.2-10459.el7.x86_64.rpm" + state: present + disable_gpg_check: yes + when: + - ansible_os_family == 'RedHat' + - ansible_distribution_major_version == '7' + - (minor_version | int <= 1) | bool + - (major_version | int <= 37) | bool + - (sub_version | int < 2) | bool + - use_compute_agent | bool \ No newline at end of file diff --git a/playbooks/roles/cloud-agent_update/tasks/main.yml b/playbooks/roles/cloud-agent_update/tasks/main.yml new file mode 100644 index 00000000..ea4d5d2a --- /dev/null +++ b/playbooks/roles/cloud-agent_update/tasks/main.yml @@ -0,0 +1,4 @@ +- include: el.yml + when: ansible_os_family == 'RedHat' and ansible_distribution == 'OracleLinux' +- include: ubuntu.yml + when: ansible_os_family == 'Debian' diff --git a/playbooks/roles/cloud-agent_update/tasks/ubuntu.yml b/playbooks/roles/cloud-agent_update/tasks/ubuntu.yml new file mode 100644 index 00000000..813e0859 --- /dev/null +++ b/playbooks/roles/cloud-agent_update/tasks/ubuntu.yml @@ -0,0 +1,47 @@ +--- +- name: Check oracle cloud agent version + shell: "snap info oracle-cloud-agent | grep installed | awk '{print $2}'" + register: version + when : use_compute_agent | bool + +- name: Download Snap package + vars: + - major_version: "{{version.stdout.split('.')[1] }}" + - minor_version: "{{version.stdout.split('.')[0] }}" + - sub_version: "{{version.stdout.split('.')[2].split('-')[0] }}" + get_url: + url: "https://objectstorage.us-phoenix-1.oraclecloud.com/p/KVgO3DVFyTiLdCJLZUNYZlYCF5mcnaTjemmMUmyOB1Ln7Vkii-llp7QoQDiRF66T/n/imagegen/b/agent_test/o/1.37.0/4/oracle-cloud-agent_1.37.2-9_amd64.snap" + dest: "/tmp/oracle-cloud-agent_1.37.2-9_amd64.snap" + when : + - (minor_version | int <= 1) | bool + - (major_version | int <= 37) | bool + - (sub_version | int < 2) | bool + - use_compute_agent | bool + + ## The ansible snap module is not upgrading properly if the package already exists. + # - name: Install OCA snap v1.37 + # vars: + # - major_version: "{{version.stdout.split('.')[1] }}" + # - minor_version: "{{version.stdout.split('.')[0] }}" + # become: true + # community.general.snap: + # classic: true + # dangerous: true + # name: "/tmp/oracle-cloud-agent_1.37.0-2_amd64.snap" + # when : + # - (minor_version | int <= 1) | bool and (minor_version | int < 37) | bool + # - use_compute_agent | bool + +- name: Snap update + vars: + - major_version: "{{version.stdout.split('.')[1] }}" + - minor_version: "{{version.stdout.split('.')[0] }}" + - sub_version: "{{version.stdout.split('.')[2].split('-')[0] }}" + + become: true + shell: "snap install --classic --dangerous /tmp/oracle-cloud-agent_1.37.2-9_amd64.snap" + when : + - (minor_version | int <= 1) | bool + - (major_version | int <= 37) | bool + - (sub_version | int < 2) | bool + - use_compute_agent | bool \ No newline at end of file diff --git a/playbooks/roles/cron/tasks/el.yml b/playbooks/roles/cron/tasks/el.yml index 0697325b..77434ba4 100755 --- a/playbooks/roles/cron/tasks/el.yml +++ b/playbooks/roles/cron/tasks/el.yml @@ -28,7 +28,7 @@ name: slurm autoscaling minute: "#*" user: '{{ ansible_user }}' - job: "/opt/oci-hpc/autoscaling/crontab/autoscale_slurm.sh >> /opt/oci-hpc/logs/crontab_slurm_`date '+\\%Y\\%m\\%d'`.log 2>&1" + job: "#/opt/oci-hpc/autoscaling/crontab/autoscale_slurm.sh >> /opt/oci-hpc/logs/crontab_slurm_`date '+\\%Y\\%m\\%d'`.log 2>&1" when: not autoscaling | bool diff --git a/playbooks/roles/cron/tasks/ubuntu.yml b/playbooks/roles/cron/tasks/ubuntu.yml index c69f9baf..834fef81 100644 --- a/playbooks/roles/cron/tasks/ubuntu.yml +++ b/playbooks/roles/cron/tasks/ubuntu.yml @@ -64,7 +64,7 @@ name: slurm autoscaling minute: "#*" user: '{{ ansible_user }}' - job: "/opt/oci-hpc/autoscaling/crontab/autoscale_slurm.sh >> /opt/oci-hpc/logs/crontab_slurm_`date '+\\%Y\\%m\\%d'`.log 2>&1" + job: "#/opt/oci-hpc/autoscaling/crontab/autoscale_slurm.sh >> /opt/oci-hpc/logs/crontab_slurm_`date '+\\%Y\\%m\\%d'`.log 2>&1" when: not autoscaling | bool - name: Create a commented Slurm monitoring cron file under /etc/cron.d diff --git a/playbooks/roles/destroy_unreachable/tasks/slurm-rack-aware.yml b/playbooks/roles/destroy_unreachable/tasks/slurm-rack-aware.yml index 4471c98c..afe0cd62 100644 --- a/playbooks/roles/destroy_unreachable/tasks/slurm-rack-aware.yml +++ b/playbooks/roles/destroy_unreachable/tasks/slurm-rack-aware.yml @@ -249,7 +249,7 @@ - name: change Node Status become: true command: "scontrol update nodename={{ item }} state=future reason=terminating" - ignore_errors: force + ignore_errors: true ignore_unreachable: True with_items: "{{unreachable_slurm_nodes}}" delegate_to: 127.0.0.1 diff --git a/playbooks/roles/destroy_unreachable/tasks/slurm.yml b/playbooks/roles/destroy_unreachable/tasks/slurm.yml index e06e77a3..6249b5ca 100644 --- a/playbooks/roles/destroy_unreachable/tasks/slurm.yml +++ b/playbooks/roles/destroy_unreachable/tasks/slurm.yml @@ -139,7 +139,7 @@ - name: change Node Status become: true command: "scontrol update nodename={{ item }} state=future reason=terminating" - ignore_errors: force + ignore_errors: true ignore_unreachable: True with_items: "{{unreachable_slurm_nodes}}" delegate_to: 127.0.0.1 diff --git a/playbooks/roles/iscsi/tasks/el.yml b/playbooks/roles/iscsi/tasks/el.yml index 00f7c4ba..9fbe71bd 100755 --- a/playbooks/roles/iscsi/tasks/el.yml +++ b/playbooks/roles/iscsi/tasks/el.yml @@ -23,6 +23,7 @@ filesystem: dev: '{{ target["devicenodes"][0] }}' fstype: xfs + when: not ('slurm_backup' in group_names) - name: Mount local volume mount: diff --git a/playbooks/roles/limits/templates/limits.j2 b/playbooks/roles/limits/templates/limits.j2 index 6c872c2a..1a5b5947 100755 --- a/playbooks/roles/limits/templates/limits.j2 +++ b/playbooks/roles/limits/templates/limits.j2 @@ -17,9 +17,9 @@ ####* soft stack 1048576 #### -{% if shape == "BM.GPU.B4.8" or shape == "BM.GPU4.8" or shape == "BM.GPU.A100-v2.8"%} -* soft nproc 10240 -* hard nproc 10240 +{% if shape == "BM.GPU.B4.8" or shape == "BM.GPU4.8" or shape == "BM.GPU.A100-v2.8" or shape == "BM.GPU.H100.8" %} +* soft nproc 40960 +* hard nproc 40960 * soft nofile 20480 * hard nofile 20480 {% else %} diff --git a/playbooks/roles/localdisk/tasks/common.yml b/playbooks/roles/localdisk/tasks/common.yml index 558ae7af..09ec1e5a 100755 --- a/playbooks/roles/localdisk/tasks/common.yml +++ b/playbooks/roles/localdisk/tasks/common.yml @@ -94,4 +94,8 @@ recurse: no when: - one_lv | bool - - lv_count.stdout == '0' \ No newline at end of file + - lv_count.stdout == '0' + +- name: rebuild initramfs if ubuntu + shell: update-initramfs -k all -u + when: ansible_facts['distribution'] == "Ubuntu" diff --git a/playbooks/roles/mpivars/tasks/ubuntu.yml b/playbooks/roles/mpivars/tasks/ubuntu.yml index e87ab94c..28a313f2 100644 --- a/playbooks/roles/mpivars/tasks/ubuntu.yml +++ b/playbooks/roles/mpivars/tasks/ubuntu.yml @@ -2,7 +2,6 @@ - name: Get the openmpi version shell: cmd: ls /usr/mpi/gcc/ - warn: false register: openmpi failed_when: false diff --git a/playbooks/roles/mysql/tasks/centos.yml b/playbooks/roles/mysql/tasks/centos.yml index 710c4a81..15f8dd30 100644 --- a/playbooks/roles/mysql/tasks/centos.yml +++ b/playbooks/roles/mysql/tasks/centos.yml @@ -1,6 +1,8 @@ --- # tasks for mariadb role +- include_vars: ol_vars.yml + - name: Create /etc/opt/oci-hpc/passwords/mysql become: true file: diff --git a/playbooks/roles/mysql/tasks/debian.yml b/playbooks/roles/mysql/tasks/debian.yml index 9d4b47be..a689d7c2 100644 --- a/playbooks/roles/mysql/tasks/debian.yml +++ b/playbooks/roles/mysql/tasks/debian.yml @@ -1,6 +1,12 @@ --- # tasks for mariadb role +- include_vars: ubuntu_vars.yml + when: ansible_distribution_major_version < "22" + +- include_vars: ubuntu-2204_vars.yml + when: ansible_distribution_major_version == "22" + - name: Create /etc/opt/oci-hpc/passwords/mysql become: true file: @@ -28,6 +34,7 @@ apt_repository: repo: deb http://archive.ubuntu.com/ubuntu bionic main state: present + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < "22" - name: Install MariaDB packages vars: @@ -83,12 +90,23 @@ - meta: flush_handlers # added to resolve error unable to connect to database,check login_user and login_password are correct or/root/.my.cnf has the credentials.Exception message: (1698, "Access denied for user ''root''@''localhost''")' - - name: Change the authentication plugin of MySQL root user to mysql_native_password - shell: mysql -u root -e 'UPDATE mysql.user SET plugin="mysql_native_password" WHERE user="root" AND host="localhost"' - - - name: Flush Privileges - shell: mysql -u root -e 'FLUSH PRIVILEGES' - + # - name: Change the authentication plugin of MySQL root user to mysql_native_password + # shell: mysql -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password" + + #- name: Flush Privileges + #shell: mysql -u root -e 'FLUSH PRIVILEGES' + + - name: Set MariaDB's root password + become: true + mysql_user: + name: 'root' + password: '{{ mysql_root_pwd }}' + host_all: yes + check_implicit_admin: yes + login_unix_socket: /run/mysqld/mysqld.sock + state: present + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == "22" + - name: Set MariaDB's root password become: true mysql_user: @@ -97,6 +115,7 @@ host_all: yes check_implicit_admin: yes state: present + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < "22" - name: Render /root/.my.cnf become: true diff --git a/playbooks/roles/mysql/tasks/el.yml b/playbooks/roles/mysql/tasks/el.yml index e76fbd6e..32defe9c 100644 --- a/playbooks/roles/mysql/tasks/el.yml +++ b/playbooks/roles/mysql/tasks/el.yml @@ -1,6 +1,8 @@ --- # tasks for mysqld role +- include_vars: ol_vars.yml + - name: Create /etc/opt/oci-hpc/passwords/mysql become: true file: diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/vars/ol_vars.yml similarity index 100% rename from playbooks/roles/mysql/defaults/main.yml rename to playbooks/roles/mysql/vars/ol_vars.yml diff --git a/playbooks/roles/mysql/vars/ubuntu-2204_vars.yml b/playbooks/roles/mysql/vars/ubuntu-2204_vars.yml new file mode 100644 index 00000000..bd163b8b --- /dev/null +++ b/playbooks/roles/mysql/vars/ubuntu-2204_vars.yml @@ -0,0 +1,25 @@ +--- +mariadb_packages: + - mariadb + - mariadb-server + - MySQL-python + +mysql_packages: + - mysql-community-server + - mysql-community-client + - MySQL-python + +mysql_packages_ol8: + - mysql-server + - mysql + - mysql-connector-python + +deb_mariadb_packages: + - mariadb-server + - mariadb-common + - python3-mysqldb + - python3-pymysql +# added above for mariadb + +mariadb_db_path: '/var/lib/mysql' +mysql_db_path: '/var/lib/mysql' diff --git a/playbooks/roles/mysql/vars/ubuntu_vars.yml b/playbooks/roles/mysql/vars/ubuntu_vars.yml new file mode 100644 index 00000000..af3189aa --- /dev/null +++ b/playbooks/roles/mysql/vars/ubuntu_vars.yml @@ -0,0 +1,25 @@ +--- +mariadb_packages: + - mariadb + - mariadb-server + - MySQL-python + +mysql_packages: + - mysql-community-server + - mysql-community-client + - MySQL-python + +mysql_packages_ol8: + - mysql-server + - mysql + - mysql-connector-python + +deb_mariadb_packages: + - mariadb-server + - mariadb-common + - python-mysqldb + - python3-pymysql +# added above for mariadb + +mariadb_db_path: '/var/lib/mysql' +mysql_db_path: '/var/lib/mysql' diff --git a/playbooks/roles/nvidia-enroot/tasks/ubuntu.yml b/playbooks/roles/nvidia-enroot/tasks/ubuntu.yml index 65a700ad..2374ac25 100644 --- a/playbooks/roles/nvidia-enroot/tasks/ubuntu.yml +++ b/playbooks/roles/nvidia-enroot/tasks/ubuntu.yml @@ -32,12 +32,12 @@ - name: execute enroot-check_*.run command: bash -c "/tmp/enroot-check_*.run --verify" - - name: + - name: set enroot_top_path set_fact: enroot_top_path_checked: "/etc/enroot/" when: " not 'nvme0n1' in hostvars[inventory_hostname].ansible_devices" - - name: + - name: set enroot_top_path set_fact: enroot_top_path_checked: "{{enroot_top_path}}" when: "'nvme0n1' in hostvars[inventory_hostname].ansible_devices" @@ -116,3 +116,4 @@ - enroot_cache - enroot_runtime + \ No newline at end of file diff --git a/playbooks/roles/nvidia_peermem/tasks/common.yml b/playbooks/roles/nvidia_peermem/tasks/common.yml index d5764cd8..17178f29 100644 --- a/playbooks/roles/nvidia_peermem/tasks/common.yml +++ b/playbooks/roles/nvidia_peermem/tasks/common.yml @@ -2,7 +2,6 @@ - name: Check if its a GPU shape shell: cmd: "curl -sH \"Authorization: Bearer Oracle\" -L http://169.254.169.254/opc/v2/instance/ | jq .shape | grep GPU" - warn: false register: shape_gpu failed_when: false diff --git a/playbooks/roles/oci-cloud-agent/tasks/el.yml b/playbooks/roles/oci-cloud-agent/tasks/el.yml index 3376bd62..5b4993ed 100644 --- a/playbooks/roles/oci-cloud-agent/tasks/el.yml +++ b/playbooks/roles/oci-cloud-agent/tasks/el.yml @@ -17,14 +17,14 @@ - name: Create configuration for core - modify params set_fact: - mydata: "{{ mydata | combine(newdata) }}" + mydata: "{{ mydata | combine(newdata, recursive=True) }}" vars: newdata: plugins: osms: disabled: true when: agent_file.stat.exists - + - name: Write back to a file copy: content: '{{ mydata | to_nice_yaml }}' diff --git a/playbooks/roles/openldap/defaults/main.yml b/playbooks/roles/openldap/defaults/main.yml index 559e6fd3..815983e7 100644 --- a/playbooks/roles/openldap/defaults/main.yml +++ b/playbooks/roles/openldap/defaults/main.yml @@ -19,3 +19,9 @@ openldap_schemas: - rfc2307bis - autoinc - ppolicy + +openldap_schemas_22: + - cosine + - inetorgperson + - rfc2307bis + - autoinc diff --git a/playbooks/roles/openldap/tasks/debian.yml b/playbooks/roles/openldap/tasks/debian.yml index 49215f6c..6504c6eb 100644 --- a/playbooks/roles/openldap/tasks/debian.yml +++ b/playbooks/roles/openldap/tasks/debian.yml @@ -143,11 +143,20 @@ command: dpkg-reconfigure -f noninteractive slapd when: not dpkgcheck.found - - name: Load OpenLDAP schemas + + - name: Load OpenLDAP schemas 20.04 command: ldapadd -c -Y EXTERNAL -H ldapi:/// -Q -f /etc/ldap/schema/{{ item }}.ldif args: creates: '/etc/ldap/slapd.d/cn=config/cn=schema/cn={?}{{ item }}.ldif' with_items: '{{ openldap_schemas }}' + when: ansible_distribution_version == '20.04' + + - name: Load OpenLDAP schemas 22.04 + command: ldapadd -c -Y EXTERNAL -H ldapi:/// -Q -f /etc/ldap/schema/{{ item }}.ldif + args: + creates: '/etc/ldap/slapd.d/cn=config/cn=schema/cn={?}{{ item }}.ldif' + with_items: '{{ openldap_schemas_22 }}' + when: ansible_distribution_version == '22.04' - name: Render OpenLDAP configuration - config template: @@ -179,7 +188,7 @@ creates: '/etc/ldap/slapd.d/cn=config.ldif' notify: restart openldap - - name: Load OpenLDAP local configuration + - name: Load OpenLDAP local configuration 20.04 command: ldapmodify -c -Y EXTERNAL -H ldapi:/// -Q -f /tmp/{{ item }} args: creates: '/etc/ldap/slapd.d/cn=config/olcDatabase={?}mdb/olcOverlay={0}memberof.ldif' @@ -190,6 +199,20 @@ - local.ldif - ppolicy.ldif - memberof.ldif + when: ansible_distribution_version == '20.04' + notify: restart openldap + + - name: Load OpenLDAP local configuration 22.04 + command: ldapmodify -c -Y EXTERNAL -H ldapi:/// -Q -f /tmp/{{ item }} + args: + creates: '/etc/ldap/slapd.d/cn=config/olcDatabase={?}mdb/olcOverlay={0}memberof.ldif' + register: result + failed_when: ( result.rc not in [ 0, 20 ] ) + with_items: + - config.ldif + - local.ldif + - memberof.ldif + when: ansible_distribution_version == '22.04' notify: restart openldap - name: Check local schemas diff --git a/playbooks/roles/openldap/vars/debian_vars.yml b/playbooks/roles/openldap/vars/debian_vars.yml index bb2fc0a6..d604763b 100644 --- a/playbooks/roles/openldap/vars/debian_vars.yml +++ b/playbooks/roles/openldap/vars/debian_vars.yml @@ -8,7 +8,7 @@ openldap_packages: - libsasl2-dev - libldap2-dev - libssl-dev - - python-pexpect + - python3-pexpect - rpcbind - nscd - libpam-ldap diff --git a/playbooks/roles/packages/tasks/centos-7.yml b/playbooks/roles/packages/tasks/centos-7.yml index 30a8dace..bfccaf3c 100644 --- a/playbooks/roles/packages/tasks/centos-7.yml +++ b/playbooks/roles/packages/tasks/centos-7.yml @@ -9,4 +9,12 @@ - python3-pip package_state: latest include_role: - name: safe_yum \ No newline at end of file + name: safe_yum + +- name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes \ No newline at end of file diff --git a/playbooks/roles/packages/tasks/debian.yml b/playbooks/roles/packages/tasks/debian.yml index d3911656..b6808410 100644 --- a/playbooks/roles/packages/tasks/debian.yml +++ b/playbooks/roles/packages/tasks/debian.yml @@ -13,3 +13,10 @@ include_role: name: safe_yum +- name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes \ No newline at end of file diff --git a/playbooks/roles/packages/tasks/el-7.yml b/playbooks/roles/packages/tasks/el-7.yml index 793cd912..d0c23143 100755 --- a/playbooks/roles/packages/tasks/el-7.yml +++ b/playbooks/roles/packages/tasks/el-7.yml @@ -13,6 +13,14 @@ name: safe_yum ignore_errors: true +- name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes + - name: install oci-cli latest version become: true pip: diff --git a/playbooks/roles/packages/tasks/main.yml b/playbooks/roles/packages/tasks/main.yml index cdc1f36a..24cc3ed3 100755 --- a/playbooks/roles/packages/tasks/main.yml +++ b/playbooks/roles/packages/tasks/main.yml @@ -8,7 +8,10 @@ when: ansible_os_family == 'RedHat' and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - include: ubuntu.yml - when: ansible_distribution == 'Ubuntu' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < '22' + +- include: ubuntu-2204.yml + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '22' - include: debian.yml when: ansible_distribution == 'Debian' \ No newline at end of file diff --git a/playbooks/roles/packages/tasks/ol-7.yml b/playbooks/roles/packages/tasks/ol-7.yml index f3380d49..7159eee4 100644 --- a/playbooks/roles/packages/tasks/ol-7.yml +++ b/playbooks/roles/packages/tasks/ol-7.yml @@ -16,6 +16,14 @@ include_role: name: safe_yum +- name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes + - name: install oci-cli latest version become: true pip: diff --git a/playbooks/roles/packages/tasks/ol-8.yml b/playbooks/roles/packages/tasks/ol-8.yml index b3733379..ad1d9877 100644 --- a/playbooks/roles/packages/tasks/ol-8.yml +++ b/playbooks/roles/packages/tasks/ol-8.yml @@ -17,6 +17,15 @@ name: safe_yum ignore_errors: true + +- name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes + - name: install oci-cli latest version become: true pip: @@ -24,4 +33,5 @@ state: latest executable: pip3 ignore_errors: yes - when: ('bastion' in group_names) \ No newline at end of file + when: ('bastion' in group_names) + diff --git a/playbooks/roles/packages/tasks/ubuntu-2204.yml b/playbooks/roles/packages/tasks/ubuntu-2204.yml new file mode 100644 index 00000000..cb11d80b --- /dev/null +++ b/playbooks/roles/packages/tasks/ubuntu-2204.yml @@ -0,0 +1,31 @@ +--- +- block: + - name: Automatically restart the services + become: true + replace: + path: /etc/needrestart/needrestart.conf + regexp: "#$nrconf{restart} = 'i';" + replace: "$nrconf{restart} = 'a';" + - name: Make sure python OpenSSL and parallel ssh is installed + vars: + package_name: + - python3-openssl + - python3-cryptography + - parted + - pssh + - pdsh + - python3-netaddr + - jq + - python3-pip + package_state: latest + include_role: + name: safe_yum + ignore_errors: true + + - name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes \ No newline at end of file diff --git a/playbooks/roles/packages/tasks/ubuntu.yml b/playbooks/roles/packages/tasks/ubuntu.yml index 408e6075..8c091071 100644 --- a/playbooks/roles/packages/tasks/ubuntu.yml +++ b/playbooks/roles/packages/tasks/ubuntu.yml @@ -14,4 +14,12 @@ package_state: latest include_role: name: safe_yum - ignore_errors: true \ No newline at end of file + ignore_errors: true + + - name: Upgrade Pip3 + become: true + pip: + name: [pip] + state: latest + executable: pip3 + ignore_errors: yes \ No newline at end of file diff --git a/playbooks/roles/rack-aware/files/node_ordering_by_rack.py b/playbooks/roles/rack-aware/files/node_ordering_by_rack.py index d86b2d07..027709e5 100644 --- a/playbooks/roles/rack-aware/files/node_ordering_by_rack.py +++ b/playbooks/roles/rack-aware/files/node_ordering_by_rack.py @@ -67,10 +67,9 @@ def write_ordered_rankfile(ordered_hosts=[],hostfile=None): for h in hosts: out = subprocess.run(["ssh "+h+" \"curl -s http://169.254.169.254/opc/v1/host/\""],stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, universal_newlines=True, check=True) x = out.stdout.splitlines() - del x[-1] - del x[0] - rackId_str = x[1].split(":")[1].replace('"','') - rackId = rackId_str.replace(' ','') + json_str = ''.join(x) + json_data = json.loads(json_str) + rackId = json_data.get("rackId", None) if rackId in r: r[rackId].append( h ) else: diff --git a/playbooks/roles/rdma-interface/tasks/debian.yml b/playbooks/roles/rdma-interface/tasks/debian.yml index 42d28db8..cfc85ba0 100644 --- a/playbooks/roles/rdma-interface/tasks/debian.yml +++ b/playbooks/roles/rdma-interface/tasks/debian.yml @@ -4,7 +4,7 @@ # Another option is to detect the mlx5_core driver, but it's harder to tell whch interface can be activated # -- name: Are we running on a new image? +- name: Are we running on an image with OCI-CN-AUTH packages? stat: path: /sbin/oci-rdma-configure register: new_image diff --git a/playbooks/roles/rdma-interface/tasks/el.yml b/playbooks/roles/rdma-interface/tasks/el.yml index e37e2ce4..395a227a 100755 --- a/playbooks/roles/rdma-interface/tasks/el.yml +++ b/playbooks/roles/rdma-interface/tasks/el.yml @@ -17,7 +17,7 @@ instance_metadata: "{{ i_metadata['content'] }}" -- name: Are we running on a new image? +- name: Are we running on an image with OCI-CN-AUTH packages? stat: path: /sbin/oci-rdma-configure register: new_image @@ -77,7 +77,7 @@ value: '2' sysctl_file: /etc/sysctl.d/80-network.conf reload: yes - when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8') and ( not new_image.stat.exists ) + when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8' or shape == 'BM.GPU.H100.8') and ( not new_image.stat.exists ) loop: "{{ ansible_mlx }}" loop_control: index_var: index @@ -88,7 +88,7 @@ value: '2' sysctl_file: /etc/sysctl.d/80-network.conf reload: yes - when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8') and ( not new_image.stat.exists ) + when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8' or shape == 'BM.GPU.H100.8') and ( not new_image.stat.exists ) loop: "{{ ansible_mlx }}" loop_control: index_var: index @@ -100,7 +100,7 @@ value: '1' sysctl_file: /etc/sysctl.d/80-network.conf reload: yes - when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8') and ( not new_image.stat.exists ) + when: ansible_mlx is defined and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8' or shape == 'BM.GPU.H100.8') and ( not new_image.stat.exists ) loop: "{{ ansible_mlx }}" loop_control: index_var: index \ No newline at end of file diff --git a/playbooks/roles/rttcc/tasks/main.yml b/playbooks/roles/rttcc/tasks/main.yml new file mode 100644 index 00000000..cb71e767 --- /dev/null +++ b/playbooks/roles/rttcc/tasks/main.yml @@ -0,0 +1,41 @@ +- name: Enable dcqcn + become: true + shell: 'mlxreg -d {{item}} -y --set "cmd_type=2" --reg_name=PPCC --indexes="local_port=1,pnat=0,lp_msb=0,algo_slot=0,algo_param_index=0"' + with_items: + - mlx5_0 + - mlx5_1 + - mlx5_3 + - mlx5_4 + - mlx5_5 + - mlx5_6 + - mlx5_7 + - mlx5_8 + - mlx5_9 + - mlx5_10 + - mlx5_12 + - mlx5_13 + - mlx5_14 + - mlx5_15 + - mlx5_16 + - mlx5_17 + +- name: Disable rttcc + become: true + shell: 'mlxreg -d {{item}} -y --set "cmd_type=1" --reg_name=PPCC --indexes="local_port=1,pnat=0,lp_msb=0,algo_slot=15,algo_param_index=0"' + with_items: + - mlx5_0 + - mlx5_1 + - mlx5_3 + - mlx5_4 + - mlx5_5 + - mlx5_6 + - mlx5_7 + - mlx5_8 + - mlx5_9 + - mlx5_10 + - mlx5_12 + - mlx5_13 + - mlx5_14 + - mlx5_15 + - mlx5_16 + - mlx5_17 \ No newline at end of file diff --git a/playbooks/roles/slurm/defaults/main.yml b/playbooks/roles/slurm/defaults/main.yml index 8b0f3f40..a4983e9c 100755 --- a/playbooks/roles/slurm/defaults/main.yml +++ b/playbooks/roles/slurm/defaults/main.yml @@ -9,4 +9,4 @@ slurm_uid: 1501 munge_gid: 1500 munge_uid: 1500 rack_aware_playbook_suffix: "{% if rack_aware|bool %}-rack-aware{% endif%}" -slurm_version: "23.02.1-1" \ No newline at end of file +slurm_version: "23.02.5-1" \ No newline at end of file diff --git a/playbooks/roles/slurm/tasks/backup_server.yml b/playbooks/roles/slurm/tasks/backup_server.yml index 096f746f..5e931304 100755 --- a/playbooks/roles/slurm/tasks/backup_server.yml +++ b/playbooks/roles/slurm/tasks/backup_server.yml @@ -55,7 +55,6 @@ become: true shell: cmd: cp /etc/munge/munge.key /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true @@ -63,7 +62,6 @@ become: true shell: cmd: chown {{ ansible_user }}:{{ ansible_user }} /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true diff --git a/playbooks/roles/slurm/tasks/common.yml b/playbooks/roles/slurm/tasks/common.yml index 4b86b7f3..804a2e63 100755 --- a/playbooks/roles/slurm/tasks/common.yml +++ b/playbooks/roles/slurm/tasks/common.yml @@ -63,8 +63,8 @@ - name: Download slurm .deb get_url: - url: "https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/VnkLhYXOSNVilVa9d24Riz1fz4Ul-KTXeK4HCKoyqv0ghW3gry3Xz8CZqloqphLw/n/hpc/b/source/o/slurm/slurm-{{slurm_version}}_amd64.deb" - dest: "{{ download_path }}/slurm_rpms" + url: "https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/VnkLhYXOSNVilVa9d24Riz1fz4Ul-KTXeK4HCKoyqv0ghW3gry3Xz8CZqloqphLw/n/hpc/b/source/o/slurm/slurm-{{slurm_version}}_{{ansible_distribution_version}}_amd64.deb" + dest: "{{ download_path }}/slurm_rpms/" when: ansible_os_family == 'Debian' and download_path == '/tmp' - name: Download slurm .rpm @@ -76,7 +76,7 @@ - name: Download slurm .deb get_url: - url: "https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/VnkLhYXOSNVilVa9d24Riz1fz4Ul-KTXeK4HCKoyqv0ghW3gry3Xz8CZqloqphLw/n/hpc/b/source/o/slurm/slurm-{{slurm_version}}_amd64.deb" + url: "https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/VnkLhYXOSNVilVa9d24Riz1fz4Ul-KTXeK4HCKoyqv0ghW3gry3Xz8CZqloqphLw/n/hpc/b/source/o/slurm/slurm-{{slurm_version}}_{{ansible_distribution_version}}_amd64.deb" dest: "{{ download_path }}/slurm_rpms" when: ansible_os_family == 'Debian' and download_path != '/tmp' delegate_to: 127.0.0.1 @@ -94,7 +94,7 @@ - name: Install .deb vars: deb_name: - - "{{ download_path }}/slurm_rpms/slurm-{{slurm_version}}_amd64.deb" + - "{{ download_path }}/slurm_rpms/slurm-{{slurm_version}}_{{ansible_distribution_version}}_amd64.deb" package_state: present include_role: name: safe_yum diff --git a/playbooks/roles/slurm/tasks/compute-rack-aware.yml b/playbooks/roles/slurm/tasks/compute-rack-aware.yml index 2d43c724..bd270e32 100755 --- a/playbooks/roles/slurm/tasks/compute-rack-aware.yml +++ b/playbooks/roles/slurm/tasks/compute-rack-aware.yml @@ -50,7 +50,6 @@ become: true shell: cmd: cp /etc/munge/munge.key /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true @@ -58,7 +57,6 @@ become: true shell: cmd: chown {{ bastion_username }}:{{ bastion_username }} /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true diff --git a/playbooks/roles/slurm/tasks/compute.yml b/playbooks/roles/slurm/tasks/compute.yml index 66ca5ed4..6dbf46cc 100755 --- a/playbooks/roles/slurm/tasks/compute.yml +++ b/playbooks/roles/slurm/tasks/compute.yml @@ -53,7 +53,6 @@ become: true shell: cmd: cp /etc/munge/munge.key /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true @@ -61,7 +60,6 @@ become: true shell: cmd: chown {{ bastion_username }}:{{ bastion_username }} /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true @@ -118,7 +116,7 @@ - name: Get nodes from Inactive Switch vars: - keyword: "{% for partition in queues %}{% for instance in partition.instance_types %}{% if instance.name == instance_type %}{{instance.instance_keyword}}{% endif %}{% endfor %}{% endfor %}" - shell: "cat {{ slurm_conf_path }}/topology.conf | grep \"SwitchName=inactive-{{queue}}-{{keyword}}\"" + shell: "cat {{ slurm_conf_path }}/topology.conf | grep \"SwitchName=inactive-{{queue}}-{{keyword}} \"" register: inactive_switch run_once: true delegate_to: 127.0.0.1 diff --git a/playbooks/roles/slurm/tasks/compute_pam.yml b/playbooks/roles/slurm/tasks/compute_pam.yml index 876e1b5a..c66d9ead 100644 --- a/playbooks/roles/slurm/tasks/compute_pam.yml +++ b/playbooks/roles/slurm/tasks/compute_pam.yml @@ -58,7 +58,7 @@ backup: yes when: ansible_distribution == 'Ubuntu' -- name: Comment pam_systemd.so in /etc/pam.d/systemd-user +- name: Comment pam_systemd.so in /etc/pam.d/systemd-user in 20.04 become: true lineinfile: path: /etc/pam.d/systemd-user @@ -66,7 +66,7 @@ line: "#session optional pam_systemd.so" state: present backup: yes - when: ansible_distribution == 'Ubuntu' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '20.04' - name: Comment pam_access.so in /etc/pam.d/common-auth become: true diff --git a/playbooks/roles/slurm/tasks/destroy-rack-aware.yml b/playbooks/roles/slurm/tasks/destroy-rack-aware.yml index dc36daf7..7f1e8846 100755 --- a/playbooks/roles/slurm/tasks/destroy-rack-aware.yml +++ b/playbooks/roles/slurm/tasks/destroy-rack-aware.yml @@ -2,7 +2,7 @@ - name: change Node Status become: true command: "scontrol update nodename={{ ansible_hostname }} state=future reason=terminating" - ignore_errors: force + ignore_errors: true ignore_unreachable: True delegate_to: 127.0.0.1 diff --git a/playbooks/roles/slurm/tasks/destroy.yml b/playbooks/roles/slurm/tasks/destroy.yml index 406b3b5b..5c58d085 100755 --- a/playbooks/roles/slurm/tasks/destroy.yml +++ b/playbooks/roles/slurm/tasks/destroy.yml @@ -112,7 +112,7 @@ - name: change Node Status become: true command: "scontrol update nodename={{ ansible_hostname }} state=future reason=terminating" - ignore_errors: force + ignore_errors: true ignore_unreachable: True delegate_to: 127.0.0.1 diff --git a/playbooks/roles/slurm/tasks/login.yml b/playbooks/roles/slurm/tasks/login.yml index 48998e34..d68da67f 100755 --- a/playbooks/roles/slurm/tasks/login.yml +++ b/playbooks/roles/slurm/tasks/login.yml @@ -49,7 +49,6 @@ become: true shell: cmd: cp /etc/munge/munge.key /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true @@ -57,7 +56,6 @@ become: true shell: cmd: chown {{ ansible_user }}:{{ ansible_user }} /tmp/munge.key - warn: false delegate_to: 127.0.0.1 run_once: true diff --git a/playbooks/roles/slurm/tasks/server.yml b/playbooks/roles/slurm/tasks/server.yml index 433152fe..9610b527 100755 --- a/playbooks/roles/slurm/tasks/server.yml +++ b/playbooks/roles/slurm/tasks/server.yml @@ -100,12 +100,10 @@ become: true shell: cmd: cp /etc/munge/munge.key /tmp/munge.key - warn: false - name: set permissions become: true shell: cmd: chown {{ ansible_user }}:{{ ansible_user }} /tmp/munge.key - warn: false - name: Create DB for accounting become: true diff --git a/playbooks/roles/slurm/templates/gres.conf.j2 b/playbooks/roles/slurm/templates/gres.conf.j2 index 8854db24..5010eb10 100644 --- a/playbooks/roles/slurm/templates/gres.conf.j2 +++ b/playbooks/roles/slurm/templates/gres.conf.j2 @@ -30,6 +30,9 @@ NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[2-3] Type=A100 Cores=[16-31] NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[4-5] Type=A100 Cores=[112-127] NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[6-7] Type=A100 Cores=[80-95] +{% elif instance.shape == "BM.GPU.H100.8"%} +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[0-3] Type=H100 Cores=[48-55] +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[4-7] Type=H100 Cores=[56-111] {% elif instance.shape == "BM.GPU.T1.2" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Name=gpu File=/dev/nvidia[0-1] Type=A10 Cores=[0-31] {% elif instance.shape == "BM.GPU.A10.4" %} diff --git a/playbooks/roles/slurm/templates/slurm.conf.j2 b/playbooks/roles/slurm/templates/slurm.conf.j2 index de6734ab..0d61eb25 100755 --- a/playbooks/roles/slurm/templates/slurm.conf.j2 +++ b/playbooks/roles/slurm/templates/slurm.conf.j2 @@ -89,6 +89,10 @@ NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boar NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=8 CoresPerSocket=16 ThreadsPerCore=1 State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} Gres=gpu:A100:8 {% elif instance.shape == "BM.GPU.A100-v2.8" and threadspercore == 2 %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket=255 ThreadsPerCore=1 State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} Gres=gpu:A100:8 +{% elif instance.shape == "BM.GPU.H100.8" and threadspercore == 1 %} +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=56 ThreadsPerCore=1 State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} Gres=gpu:H100:8 +{% elif instance.shape == "BM.GPU.H100.8" and threadspercore == 2 %} +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=56 ThreadsPerCore=2 State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} Gres=gpu:H100:8 {% elif instance.shape == "BM.GPU.T1.2" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket=32 ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} Gres=gpu:A10:2 {% elif instance.shape == "BM.GPU.A10.4" %} @@ -101,6 +105,8 @@ NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boar NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket={{instance.instance_pool_ocpus}} ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif instance.shape == "VM.Standard.E4.Flex" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket={{instance.instance_pool_ocpus}} ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} +{% elif instance.shape == "VM.Standard.E5.Flex" %} +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket={{instance.instance_pool_ocpus}} ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif instance.shape == "VM.Optimized3.Flex" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=1 CoresPerSocket={{instance.instance_pool_ocpus}} ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif instance.shape == "VM.Standard3.Flex" %} @@ -125,6 +131,8 @@ NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boar NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=18 ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif instance.shape == "BM.HPC.E5.144" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=72 ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} +{% elif instance.shape == "BM.Standard.E5.192" %} +NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=96 ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif instance.shape == "BM.Optimized3.36" %} NodeName={{partition.name}}-{{instance.instance_keyword}}-node-[1-{{size}}] Boards=1 SocketsPerBoard=2 CoresPerSocket=18 ThreadsPerCore={{threadspercore}} State=FUTURE Features={% if instance.shape != instance.name%}{{ instance.shape }},{% endif %}{{ instance.name }} {% elif "VM.Standard2." in instance.shape %} diff --git a/playbooks/roles/ssh/tasks/common.yml b/playbooks/roles/ssh/tasks/common.yml index 496a8dc4..41872c6d 100644 --- a/playbooks/roles/ssh/tasks/common.yml +++ b/playbooks/roles/ssh/tasks/common.yml @@ -7,7 +7,7 @@ group: root mode: '0644' -- name: Install ssh keys on all nodes +- name: Install private ssh key on all nodes copy: dest: "/home/{{ ansible_user }}/.ssh/id_rsa" src: "/home/{{ bastion_username }}/.ssh/{{ item }}" @@ -17,13 +17,12 @@ with_items: - cluster.key -- name: Generate an OpenSSL public key in OpenSSH v2 format - community.crypto.openssl_publickey: - path: "/home/{{ ansible_user }}/.ssh/id_rsa.pub" - privatekey_path: "/home/{{ ansible_user }}/.ssh/id_rsa" - format: OpenSSH - state: present +- name: Install public ssh key on all nodes + copy: + dest: "/home/{{ ansible_user }}/.ssh/id_rsa.pub" + src: "/home/{{ bastion_username }}/.ssh/{{ item }}" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" mode: '0644' - register: public_key + with_items: + - id_rsa.pub \ No newline at end of file diff --git a/playbooks/roles/tuned/tasks/main.yml b/playbooks/roles/tuned/tasks/main.yml index 637e8bae..15793bee 100644 --- a/playbooks/roles/tuned/tasks/main.yml +++ b/playbooks/roles/tuned/tasks/main.yml @@ -1,2 +1,2 @@ - include: el-7.yml - when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7' and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8') + when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7' and (shape == 'BM.GPU.B4.8' or shape == 'BM.GPU4.8' or shape == 'BM.GPU.A100-v2.8' or shape == 'BM.GPU.H100.8') diff --git a/playbooks/roles/weka_client/tasks/main.yml b/playbooks/roles/weka_client/tasks/main.yml new file mode 100755 index 00000000..6d4b0887 --- /dev/null +++ b/playbooks/roles/weka_client/tasks/main.yml @@ -0,0 +1,327 @@ +--- +# tasks file for weka_client + + +# +# GET INSTANCE/VNIC METADATA +# +- name: Get instance metadata + uri: + url: http://169.254.169.254/opc/v2/instance/ + method: GET + return_content: true + status_code: 200 + body_format: json + headers: + Authorization: Bearer Oracle + register: instance_metadata + +- name: Get vnic metadata + uri: + url: http://169.254.169.254/opc/v1/vnics/ + method: GET + return_content: true + status_code: 200 + body_format: json + headers: + Authorization: Bearer Oracle + register: vnic_metadata + +- name: Set vnic facts from metadata + set_fact: + private_ip: "{{ vnic_metadata.json[0].privateIp }}" + subnet_cidr_block: "{{ vnic_metadata.json[0].subnetCidrBlock[-2:] }}" + gateway: "{{ vnic_metadata.json[0].virtualRouterIp }}" + +- name: Set vnic facts from metadata + set_fact: + private_ip_2: "{{ vnic_metadata.json[1].privateIp }}" + subnet_cidr_block_2: "{{ vnic_metadata.json[1].subnetCidrBlock[-2:] }}" + gateway_2: "{{ vnic_metadata.json[1].virtualRouterIp }}" + when: shape == "VM.Standard3.Flex" or shape == "VM.Standard.E4.Flex" or shape == "VM.Optimized3.Flex" + + +# +# SYSTEM CONFIG +# +- name: Change the /etc/os-release to install Weka + become: true + replace: + path: /etc/os-release + regexp: '^NAME="Oracle Linux Server"' + replace: 'NAME="Red Hat Enterprise Linux"' + +- name: Rt already set check + shell: "/usr/sbin/ip rule | wc -l " + register: rt_check + +- name: Mkdir for routing tables and scripts + file: + path: /tmp/weka + state: directory + mode: 0755 + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) +- name: Copy files + copy: + src: files/ + dest: /tmp/weka/ + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + +- name: Copy IP route rules + copy: + src: /tmp/weka/rt_tables + dest: /etc/iproute2/rt_tables + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + + +- name: Get interface names + shell: ip a | grep BROADCAST | awk '{ print $2 }' | grep -v 'ens\|docker\|lo' | sed 's/://' + register: interface_names + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + +- name: Create list of interface names + copy: + dest: "/tmp/interface_names" + content: | + {{ interface_names.stdout }} + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + +- name: Loop over NIC names and set up routing tables and rules + shell: | + TABLE=0 + + while read -r NIC; do + IP='' + TABLE=$((TABLE+1)) + + while read -r line; do + if [[ $line =~ IPADDR=([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) ]]; then + IP=${BASH_REMATCH[1]} + fi + done < "/etc/sysconfig/network-scripts/ifcfg-$line" + + echo "192.168.0.0/16 dev $NIC src $IP table net$TABLE" > /etc/sysconfig/network-scripts/route-$line + echo "table net$TABLE from $IP" > /etc/sysconfig/network-scripts/rule-$line + done < /tmp/interface_names + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + +- name: Reset Network Interfaces + shell: "for nic in $(cat /tmp/interface_names); do sudo ifdown $nic; sudo ifup $nic; done" + when: rt_check.stdout | int != 19 and ( + (shape == "BM.GPU4.8") or + (shape == "BM.GPU.B4.8") or + (shape == "BM.GPU.A100-v2.8") or + (shape == "BM.GPU.H100.8") ) + +# +# SECONDARY VNIC SETUP +# + +- name: Get VNIC Number + shell: "export LC_ALL=C.UTF-8;export LANG=C.UTF-8; /usr/bin/oci compute instance list-vnics --auth instance_principal --instance-id {{instance_metadata.json.id}} | jq '.[] | length'" + register: vnic_number + when: shape == "VM.Standard3.Flex" or shape == "VM.Standard.E4.Flex" or shape == "VM.Optimized3.Flex" + +- name: attach secondary vnic + shell: "export LC_ALL=C.UTF-8;export LANG=C.UTF-8; oci compute instance attach-vnic --instance-id {{instance_metadata.json.id}} --vnic-display-name 'weka' --subnet-id {{secondary_vnic_subnet}} --auth instance_principal" + when: (shape == "VM.Standard3.Flex" or shape == "VM.Standard.E4.Flex" or shape == "VM.Optimized3.Flex") and vnic_number.stdout | int < 2 + register: add_vnic + delegate_to: 127.0.0.1 + +- name: Sleep + pause: + seconds: 30 + when: (shape == "VM.Standard3.Flex" or shape == "VM.Standard.E4.Flex" or shape == "VM.Optimized3.Flex") and vnic_number.stdout | int < 2 + + +# +# WEKA INSTALL +# + +- name: Check is Weka installed + command: command -v weka >/dev/null 2>&1 + register: installed + ignore_errors: true + +- name: Download Weka installer + get_url: + url: "http://{{ dist_server }}:14000/dist/v1/install" + dest: /tmp/weka_client_install + mode: 0755 + when: installed is failed + register: download + +- name: Execute the Weka installer + shell: /tmp/weka_client_install + when: download.changed + +- name: Remove the weka_client_install + file: + path: /tmp/weka_client_install + state: absent + +- name: Change the /etc/wekaio/service.conf to install Weka + become: true + replace: + path: /etc/wekaio/service.conf + regexp: '^isolate_cpusets=true' + replace: 'isolate_cpusets=false' + + +# +# MOUNT SETUP +# + +- name: Create Weka DPDK mount points + file: + path: "{{ item.mountpoint }}" + owner: opc + group: "${privilege_group_name}" + mode: 0775 + state: directory + loop: "{{ dpdk_fs | default([]) }}" + when: dpdk_fs is defined + + +- name: Create Weka UDP mount points + file: + path: "{{ item.mountpoint }}" + owner: opc + group: "${privilege_group_name}" + mode: 0775 + state: directory + loop: "{{ udp_fs }}" + when: upd_fs is defined + + +- name: Create Weka NFS mount points + file: + path: "{{ item.mountpoint }}" + owner: opc + group: "${privilege_group_name}" + mode: 0775 + state: directory + loop: "{{ weka_nfs }}" + when: weka_nfs is defined + + +- name: Mount Weka DPDK Filesystems GPU Nodes + mount: + path: "{{ item.mountpoint }}" + src: "{{ backends | join(',') }}/{{item.name}}" + fstype: wekafs + opts: "{{ dpdk_opts }},net={{ ansible_default_ipv4.alias }}{{ item.opts }}" + state: mounted + when: (shape == "BM.GPU4.8" or shape == "BM.GPU.B4.8" or shape == "BM.GPU.A100-v2.8" or shape == "BM.GPU.H100.8") + and dpdk_fs is defined + loop: "{{ dpdk_fs }}" + +# TODO: Add additonal debug info for failing mounts + +- name: Mount Weka DPDK Filesystems to VM nodes + mount: + path: "{{ item.mountpoint }}" + src: "{{ data_backends | join(',') }}/{{item.name}}" + fstype: wekafs + opts: "{{ dpdk_opts }},net=ens5/{{ private_ip_2 }}/{{ subnet_cidr_block_2 }}/{{ gateway_2}}{{ item.opts }}" + state: mounted + when: (shape == "VM.Standard3.Flex" or shape == "VM.Standard.E4.Flex" or shape == "VM.Optimized3.Flex") + and dpdk_fs is defined + loop: "{{ dpdk_fs }}" + +- name: Mount Weka UDP to all nodes + mount: + path: "{{ item.mountpoint }}" + src: "{{ backends | join(',') }}/{{item.name}}" + fstype: wekafs + opts: "{{ upd_opts }},net=udp,{{ item.opts }}" + state: mounted + when: udp_fs is defined + loop: "{{ udp_fs }}" + +- name: Set NFS Mount + set_fact: + nfsmount: "{{ nfs_backends | random }}" + when: weka_nfs is defined and nfs_backends is defined + +- name: Mount Weka-NFS Filesystems to nodes + mount: + path: "{{ item.mountpoint }}" + src: "{{ nfsmount }}:/{{ item.name }}" + fstype: nfs + state: mounted + when: weka_nfs is defined + loop: "{{ weka_nfs }}" + + +# +# WEKA CLEAN-UP SCRIPTS +# + +- name: Create Pre-Shutdowm Weka Script + copy: + dest: /opt/pre-shutdown-weka.sh + mode: 0755 + content: | + #!/bin/bash + + {% for fs in dpdk_fs %} + if mountpoint -q "{{ fs.mountpoint }}" ; then + echo "{{ fs.mountpoint }} is mounted, unmounting" + umount "{{ fs.mountpoint }}" + else + echo "{{ fs.mountpoint }} is already umounted" + fi + {% endfor %} + + {% for fs in udp_fs %} + if mountpoint -q "{{ fs.mountpoint }}" ; then + echo "{{ fs.mountpoint }} is mounted, unmounting" + umount "{{ fs.mountpoint }}" + else + echo "{{ fs.mountpoint }} is already umounted" + fi + {% endfor %} + +- name: Create Pre-shutdown-weka Systemd Service File + template: + src: templates/pre-shutdown-weka.service.j2 + dest: /lib/systemd/system/pre-shutdown-weka.service + +- name: Enable Pre-shutdown-weka Service + systemd: + name: pre-shutdown-weka + enabled: true + daemon_reload: yes + +- name: Change the /etc/os-release to install Weka + become: true + replace: + path: /etc/os-release + regexp: '^NAME="Red Hat Enterprise Linux"' + replace: 'NAME="Oracle Linux Server"' + diff --git a/playbooks/site.yml b/playbooks/site.yml index 0ee6eb8f..3bb6f837 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -50,12 +50,15 @@ become: true gather_facts: true tasks: + - include_role: + name: cloud-agent_update + when: cluster_network|bool and use_compute_agent|default(false)|bool - include_role: name: oci-cn-auth - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: rdma-interface - when: cluster_network|bool + when: cluster_network|bool and not use_compute_agent|default(false)|bool - include_role: name: nvidia_peermem @@ -101,6 +104,8 @@ options: "" lock: "none" when: add_nfs|bool and home_fss|bool + - include_role: + name: passwords - hosts: bastion @@ -160,9 +165,7 @@ export_name: "cluster" local_path: "/export/cluster" iscsi_ip: "{{ bastion_mount_ip }}" - tasks: - - include_role: - name: passwords + tasks: - include_role: name: iscsi when: bastion_block|default(false)|bool @@ -172,17 +175,26 @@ - include_role: name: mysql -- hosts: slurm_backup, login +- hosts: slurm_backup become: true vars: iscsi_ip: "{{ bastion_mount_ip }}" + local_path: "/mnt/nfs_backup" tasks: - - include_role: - name: passwords - include_role: name: iscsi when: bastion_block|default(false)|bool +- hosts: login + become: true + vars: + iscsi_ip: "{{ login_mount_ip }}" + local_path: "/mnt/block" + tasks: + - include_role: + name: iscsi + when: login_block|default(false)|bool + - hosts: nfs become: true vars: diff --git a/queues.conf b/queues.conf index 2c43b0a2..351846b2 100644 --- a/queues.conf +++ b/queues.conf @@ -21,7 +21,7 @@ targetCompartment: ${targetCompartment} boot_volume_size: ${boot_volume_size} use_marketplace_image: ${use_marketplace_image} - use_old_marketplace_image: ${use_old_marketplace_image} + use_compute_agent: ${use_compute_agent} instance_pool_ocpus: ${instance_pool_ocpus} instance_pool_memory: ${instance_pool_memory} instance_pool_custom_memory: ${instance_pool_custom_memory} @@ -45,7 +45,7 @@ targetCompartment: ${targetCompartment} boot_volume_size: ${boot_volume_size} use_marketplace_image: ${use_marketplace_image} - use_old_marketplace_image: ${use_old_marketplace_image} + use_compute_agent: ${use_compute_agent} instance_pool_ocpus: ${instance_pool_ocpus} instance_pool_memory: ${instance_pool_memory} instance_pool_custom_memory: ${instance_pool_custom_memory} diff --git a/samples/gpu/H100-topology-container.xml b/samples/gpu/H100-topology-container.xml new file mode 100644 index 00000000..85975887 --- /dev/null +++ b/samples/gpu/H100-topology-container.xml @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/gpu/H100-topology.xml b/samples/gpu/H100-topology.xml new file mode 100644 index 00000000..3dcce372 --- /dev/null +++ b/samples/gpu/H100-topology.xml @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/gpu/nccl_run_allreduce_H100.sbatch b/samples/gpu/nccl_run_allreduce_H100.sbatch new file mode 100644 index 00000000..830870c4 --- /dev/null +++ b/samples/gpu/nccl_run_allreduce_H100.sbatch @@ -0,0 +1,68 @@ +#!/bin/bash +#SBATCH --job-name=nccl-allreduce-slurm +#SBATCH --nodes=2 +#SBATCH --gpus-per-node=8 +#SBATCH --ntasks-per-node=8 +#SBATCH --exclusive +export PMI_DEBUG=1 + + +cd /nfs/cluster +mkdir $SLURM_JOB_ID +cd $SLURM_JOB_ID + +MACHINEFILE="hostfile" + +scontrol show hostnames $SLURM_JOB_NODELIST > $MACHINEFILE +echo MACHINEFILE +cat $MACHINEFILE + +source /etc/os-release + +mpivars_path=`ls /usr/mpi/gcc/openmpi-*/bin/mpivars.sh` + +if [[ "$mpivars_path" == "" ]]; then + mpivars_path=`ls /opt/openmpi-*/bin/mpivars.sh` +fi + +if [[ "$mpivars_path" == "" ]]; then + echo "Could not find MPIPATH"; exit; fi + +source $mpivars_path + +shape=`curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ | jq .shape` +if [ $shape == \"BM.GPU.H100.8\" ] +then + var_UCX_NET_DEVICES=eth0 + var_NCCL_IB_HCA="=mlx5_0,mlx5_1,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_9,mlx5_10,mlx5_12,mlx5_13,mlx5_14,mlx5_15,mlx5_16,mlx5_17" +else + echo "Use the appropriate nccl test run script for non H100 nodes" +fi + + mpirun --mca pml ucx \ + --bind-to numa \ + -npernode 8 \ + --mca coll ^hcoll \ + -x NCCL_CROSS_NIC=0 \ + -x NCCL_SOCKET_NTHREADS=16 \ + -x NCCL_DEBUG=WARN \ + -x NCCL_CUMEM_ENABLE=0 \ + -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \ + -x NCCL_IB_QPS_PER_CONNECTION=16 \ + -x NCCL_IB_GID_INDEX=3 \ + -x NCCL_IB_TC=41 \ + -x NCCL_IB_SL=0 \ + -x NCCL_IB_TIMEOUT=22 \ + -x NCCL_NET_PLUGIN=none \ + -x HCOLL_ENABLE_MCAST_ALL=0 \ + -x coll_hcoll_enable=0 \ + -x UCX_TLS=tcp \ + -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \ + -x RX_QUEUE_LEN=8192 \ + -x IB_RX_QUEUE_LEN=8192 \ + -x NCCL_SOCKET_IFNAME=eth0 \ + -x NCCL_ALGO=auto \ + -x NCCL_IGNORE_CPU_AFFINITY=1 \ + -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \ + -x NCCL_TOPO_FILE=~/H100-topology.xml \ + --np $((SLURM_NNODES*SLURM_NTASKS_PER_NODE)) --hostfile $MACHINEFILE /opt/oci-hpc/nccl-test/build/all_reduce_perf -b1G -e10G -i$((1024*1024*1024*9)) -n 100 \ No newline at end of file diff --git a/samples/gpu/nccl_run_allreduce_H100.sh b/samples/gpu/nccl_run_allreduce_H100.sh new file mode 100644 index 00000000..520f125d --- /dev/null +++ b/samples/gpu/nccl_run_allreduce_H100.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -e + +# number of times to run the nccl test to stress the GPUs and RDMA network. This is different from -n iterations parameter of nccl allreduce which is set below using $iter +max=$1 + +# This assume, the hostfile passed is already ordered based on their rackId +if [ -n "$2" ]; then + hostfile=$2 +else + hostfile="/etc/opt/oci-hpc/hostfile.tcp" +fi + +echo INPUTFILE +cat $hostfile + +# The number of GPUs to use for the test. Has to be multiplier of 8. If not passed, all GPUs will be used. +if [ -n "$3" ]; then + np=$3 +else + np=$((`less $hostfile | wc -l` * 8 )) +fi + +logfile="nccl_run_allreduce.sh.log" + +for x in $(seq 1 1 $max) +do + + echo $x + echo $x >> $logfile + date >> $logfile + + hostfile=$hostfile; np=$np ; iter=20; + + mpivars_path=`ls /usr/mpi/gcc/openmpi-*/bin/mpivars.sh` + + if [[ "$mpivars_path" == "" ]]; then + mpivars_path=`ls /opt/openmpi-*/bin/mpivars.sh` + fi + + if [[ "$mpivars_path" == "" ]]; then + echo "Could not find MPIPATH"; exit; fi + + source $mpivars_path + + first_node=`head $hostfile -n 1` + shape=`ssh $first_node 'curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/' | jq .shape` + if [ $shape == \"BM.GPU.H100.8\" ] + then + var_UCX_NET_DEVICES=eth0 + var_NCCL_IB_HCA="=mlx5_0,mlx5_1,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_9,mlx5_10,mlx5_12,mlx5_13,mlx5_14,mlx5_15,mlx5_16,mlx5_17" + else + echo "Use the appropriate nccl test run script for non H100 nodes" + fi + + mpirun --mca pml ucx \ + --bind-to numa \ + -npernode 8 \ + --mca coll ^hcoll \ + -x NCCL_CROSS_NIC=0 \ + -x NCCL_SOCKET_NTHREADS=16 \ + -x NCCL_DEBUG=WARN \ + -x NCCL_CUMEM_ENABLE=0 \ + -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \ + -x NCCL_IB_QPS_PER_CONNECTION=16 \ + -x NCCL_IB_GID_INDEX=3 \ + -x NCCL_IB_TC=41 \ + -x NCCL_IB_SL=0 \ + -x NCCL_IB_TIMEOUT=22 \ + -x NCCL_NET_PLUGIN=none \ + -x HCOLL_ENABLE_MCAST_ALL=0 \ + -x coll_hcoll_enable=0 \ + -x UCX_TLS=tcp \ + -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \ + -x RX_QUEUE_LEN=8192 \ + -x IB_RX_QUEUE_LEN=8192 \ + -x NCCL_SOCKET_IFNAME=eth0 \ + -x NCCL_ALGO=auto \ + -x NCCL_IGNORE_CPU_AFFINITY=1 \ + -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \ + -x NCCL_TOPO_FILE=~/H100-topology.xml \ + --np $np --hostfile $hostfile /opt/oci-hpc/nccl-test/build/all_reduce_perf -b1G -e10G -i$((1024*1024*1024*9)) -n 100 >> $logfile + + tail -n 32 $logfile + + +done \ No newline at end of file diff --git a/samples/gpu/ping.sh b/samples/gpu/ping.sh new file mode 100644 index 00000000..25cf4797 --- /dev/null +++ b/samples/gpu/ping.sh @@ -0,0 +1,9 @@ +#!/usr/bin/bash +f=$(mktemp) +HOST=$1 +ssh $HOST /usr/sbin/ip -j addr | jq -r '.[] | select(.ifname | test("rdma")) | .ifname + " " + .addr_info[0].local' > $f +while read -r l ; do + i=$(echo $l | awk '{print $1}') + ip=$(echo $l | awk '{print $2}') + ping -qI $i $ip -c1 ; done < $f +rm -rf $f \ No newline at end of file diff --git a/samples/prep_sample_files.sh b/samples/prep_sample_files.sh index 0b9d78fb..2c1f43a8 100644 --- a/samples/prep_sample_files.sh +++ b/samples/prep_sample_files.sh @@ -7,7 +7,8 @@ do sudo chmod +x $directory/*.sh done; -cp nccl_compile/compile.sh /home/opc/ -cp gpu/*.sbatch /home/opc/ -cp /opt/oci-hpc/bin/node_ordering_by_rack.py /home/opc/ +cp nccl_compile/compile.sh ~ +cp gpu/*.sbatch ~ +cp gpu/H100* ~ +cp /opt/oci-hpc/bin/node_ordering_by_rack.py ~ diff --git a/schema.yaml b/schema.yaml index 42a9890d..037e685b 100755 --- a/schema.yaml +++ b/schema.yaml @@ -56,16 +56,15 @@ variableGroups: - ${node_count} - ${hyperthreading} - ${boot_volume_size} + - ${use_compute_agent} - ${use_marketplace_image} - - ${use_old_marketplace_image} - ${compute_username} - ${marketplace_listing} - - ${old_marketplace_listing} - ${unsupported} - ${compute_image_compartment} - ${image} - ${image_ocid} - - title: "Additionnal Login Node" + - title: "Additional Login Node" variables: - ${login_node} - ${login_ad} @@ -75,11 +74,8 @@ variableGroups: - ${login_custom_memory} - ${login_memory} - ${login_boot_volume_size} - - ${use_standard_image_login} - ${use_marketplace_image_login} - - ${use_old_marketplace_image_login} - ${marketplace_listing_login} - - ${old_marketplace_listing_login} - ${unsupported_login} - ${login_image_compartment} - ${custom_login_image} @@ -121,7 +117,8 @@ variableGroups: - ${fss_ad} - title: "Advanced bastion options" variables: - - ${use_standard_image} + - ${use_marketplace_image_bastion} + - ${marketplace_listing_bastion} - ${unsupported_bastion} - ${bastion_image_compartment} - ${custom_bastion_image} @@ -184,7 +181,6 @@ variableGroups: - ${scratch_nfs_export} - ${scratch_nfs_mount} - ${marketplace_listing_id} - - ${old_marketplace_listing_id} - ${marketplace_listing_id_GPU} - ${marketplace_listing_id_HPC} - ${ssh_cidr} @@ -288,6 +284,9 @@ variables: - eq: - ${bastion_shape} - "VM.Standard.E4.Flex" + - eq: + - ${bastion_shape} + - "VM.Standard.E5.Flex" - eq: - ${bastion_shape} - "VM.Optimized3.Flex" @@ -332,6 +331,9 @@ variables: - eq: - ${bastion_shape} - "VM.Standard.E4.Flex" + - eq: + - ${bastion_shape} + - "VM.Standard.E5.Flex" - eq: - ${bastion_shape} - "VM.Standard.A1.Flex" @@ -358,6 +360,9 @@ variables: - eq: - ${bastion_shape} - "VM.Standard.E4.Flex" + - eq: + - ${bastion_shape} + - "VM.Standard.E5.Flex" - eq: - ${bastion_shape} - "VM.Standard.A1.Flex" @@ -369,31 +374,39 @@ variables: required: true bastion_object_storage_par: title: Create Object Storage PAR - description: "Create a PAR (i.e. Pre-Authenticated Request), so that user could use that PAR to upload monitoring metrics to - to Object Storage and share the URL with OCI service teams." - type: boolean - default: true - use_standard_image: + description: "Create a PAR (i.e. Pre-Authenticated Request), so that user could use that PAR to upload monitoring metrics to Object Storage and share the URL with OCI service teams." type: boolean - title: "use standard bastion image" - description: > - "Use standard bastion image (Oracle Linux)" default: true - visible: true unsupported_bastion: title: "Use unsupported image" description: "Custom image ID for Bastion" type: boolean default: false - visible: - or: - - not: - - ${use_standard_image} - - not: - - eq: - - ${compute_username} - - "opc" + visible: + not: + - ${use_marketplace_image_bastion} + + use_marketplace_image_bastion: + type: boolean + title: "use marketplace image" + description: "Use marketplace image, otherwise provide custom image OCID" + default: true + visible: true + + + marketplace_listing_bastion: + type: enum + title: "Image version" + description: "Marketplace listing to use" + required: true + enum: + - "HPC_OL7" + - "HPC_OL8" + - "GPU_OL7" + - "GPU_OL8" + default: "HPC_OL7" + visible: ${use_marketplace_image_bastion} bastion_username: title: "Default username for bastion" @@ -401,29 +414,14 @@ variables: type: string default: "opc" required: true - visible: - or: - - not: - - ${use_standard_image} - - not: - - eq: - - ${compute_username} - - "opc" + visible: true unsupported_bastion_image: title: "Image OCID" description: "Custom image ID for compute nodes. Please note that only Oracle Linux 7 and Ubuntu 20.04 are supported as bastion image at this moment." type: string required: true - visible: - and: - - or: - - not: - - ${use_standard_image} - - not: - - eq: - - ${compute_username} - - "opc" + visible: ${unsupported_bastion} default: "image.ocid" bastion_image_compartment: @@ -432,15 +430,10 @@ variables: default: ${targetCompartment} visible: and: - - or: - - not: - - ${use_standard_image} - - not: - - eq: - - ${compute_username} - - "opc" - not: - ${unsupported_bastion} + - not: + - ${use_marketplace_image_bastion} required: true custom_bastion_image: @@ -449,17 +442,12 @@ variables: type: oci:core:image:id dependsOn: compartmentId: ${bastion_image_compartment} - visible: + visible: and: - - or: - - not: - - ${use_standard_image} - - not: - - eq: - - ${compute_username} - - "opc" - not: - - ${unsupported_bastion} + - ${unsupported_bastion} + - not: + - ${use_marketplace_image_bastion} required: true bastion_boot_volume_size: @@ -467,7 +455,7 @@ variables: required: true minimum: 50 title: "Size of the boot volume in GB" - default: 50 + default: 100 bastion_boot_volume_backup: type: boolean @@ -584,6 +572,7 @@ variables: - "BM.GPU4.8" - "BM.GPU.B4.8" - "BM.GPU.A100-v2.8" + - "BM.GPU.H100.8" - "BM.Optimized3.36" - "BM.HPC.E5.144" default: "BM.HPC2.36" @@ -627,6 +616,9 @@ variables: - eq: - ${instance_pool_shape} - "VM.Standard.E4.Flex" + - eq: + - ${instance_pool_shape} + - "VM.Standard.E5.Flex" - eq: - ${instance_pool_shape} - "VM.Standard.A1.Flex" @@ -670,6 +662,9 @@ variables: - eq: - ${instance_pool_shape} - "VM.Standard.E4.Flex" + - eq: + - ${instance_pool_shape} + - "VM.Standard.E5.Flex" - eq: - ${instance_pool_shape} - "VM.Standard.A1.Flex" @@ -696,6 +691,9 @@ variables: - eq: - ${instance_pool_shape} - "VM.Standard.E4.Flex" + - eq: + - ${instance_pool_shape} + - "VM.Standard.E5.Flex" - eq: - ${instance_pool_shape} - "VM.Standard.A1.Flex" @@ -704,7 +702,8 @@ variables: - "VM.Standard3.Flex" - and: - ${instance_pool_custom_memory} - - ${cluster_network} + - not: + - ${cluster_network} required: true node_count: @@ -726,7 +725,7 @@ variables: required: true minimum: 50 title: "Size of the boot volume in GB" - default: 50 + default: 100 description: "Boot volume size in GB of each compute node" use_marketplace_image: @@ -735,15 +734,6 @@ variables: description: "Use marketplace image, otherwise provide custom image OCID" default: true - use_old_marketplace_image: - type: boolean - title: "use older marketplace images" - description: "Images prior to September 2021" - default: false - visible: - and: - - ${use_marketplace_image} - marketplace_listing: type: enum title: "Image version" @@ -752,29 +742,16 @@ variables: enum: - "HPC_OL7" - "HPC_OL8" - - "GPU" + - "GPU_OL7" + - "GPU_OL8" default: "HPC_OL7" - visible: - and: - - ${use_marketplace_image} - - not: - - ${use_old_marketplace_image} + visible: ${use_marketplace_image} - old_marketplace_listing: - type: enum - title: "Image version" - description: "Marketplace listing to use" - required: true - enum: - - "1. Oracle Linux 7.9 OFED 5.3-1.0.0.1 RHCK 20210607" - - "2. Oracle Linux 7.8 OFED 5.0-1.0.0.0 UEK 20200826" - - "3. Oracle Linux 7.7 OFED 4.4-2.0.7.0 UEK 20200229" - - "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" - default: "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" - visible: - and: - - ${use_marketplace_image} - - ${use_old_marketplace_image} + use_compute_agent: + type: boolean + title: "use compute agent" + description: "Select if your image has the OCA agent rather than the oci-cn-auth package. The new marketplace images need the compute agent enabled." + default: true compute_image_compartment: title: "compute image compartment" @@ -1023,8 +1000,8 @@ variables: rdma_subnet: type: string title: "RDMA subnet IP range" - default: "192.168.0.0/16" - description: "Must be at least the same size as private subnet for HPC and at least 16 times the size of the private subnet for GPUs" + default: "10.224.0.0/12" + description: "Must be at least the same size as private subnet for HPC and at least 16 times the size of the private subnet for GPUs, currently cannnot be modified with the compute agent" required: true private_subnet: type: string @@ -1390,6 +1367,9 @@ variables: - eq: - ${login_shape} - "VM.Standard.E4.Flex" + - eq: + - ${login_shape} + - "VM.Standard.E5.Flex" - eq: - ${login_shape} - "VM.Optimized3.Flex" @@ -1436,6 +1416,9 @@ variables: - eq: - ${login_shape} - "VM.Standard.E4.Flex" + - eq: + - ${login_shape} + - "VM.Standard.E5.Flex" - eq: - ${login_shape} - "VM.Standard.A1.Flex" @@ -1463,6 +1446,9 @@ variables: - eq: - ${login_shape} - "VM.Standard.E4.Flex" + - eq: + - ${login_shape} + - "VM.Standard.E5.Flex" - eq: - ${login_shape} - "VM.Standard.A1.Flex" @@ -1512,22 +1498,17 @@ variables: - and: - ${login_block} - ${login_node} - use_standard_image_login: - type: boolean - title: "use standard login image" - description: > - "Use standard login image (Oracle Linux)" - default: true - visible: ${login_node} unsupported_login: title: "Use unsupported image" description: "Custom image ID for Login Node" type: boolean default: false - visible: - not: - - ${use_standard_image_login} + visible: + and: + - ${login_node} + - not: + - ${use_marketplace_image_login} login_image_compartment: title: "login image compartment" @@ -1535,8 +1516,7 @@ variables: default: ${targetCompartment} visible: and: - - not: - - ${use_standard_image_login} + - ${login_node} - not: - ${unsupported_login} - not: @@ -1551,8 +1531,7 @@ variables: compartmentId: ${login_image_compartment} visible: and: - - not: - - ${use_standard_image_login} + - ${login_node} - not: - ${unsupported_login} - not: @@ -1566,8 +1545,6 @@ variables: visible: and: - ${unsupported_login} - - not: - - ${use_standard_image_login} - not: - ${use_marketplace_image_login} default: "image.ocid" @@ -1578,28 +1555,14 @@ variables: type: string default: "opc" required: true - visible: - not: - - ${use_standard_image_login} + visible: ${login_node} use_marketplace_image_login: type: boolean title: "use marketplace image" description: "Use marketplace image, otherwise provide custom image OCID" default: true - visible: - not: - - ${use_standard_image_login} - use_old_marketplace_image_login: - type: boolean - title: "use older marketplace images" - description: "Images prior to September 2021" - default: false - visible: - and: - - ${use_marketplace_image_login} - - not: - - ${use_standard_image_login} + visible: ${login_node} marketplace_listing_login: type: enum @@ -1609,30 +1572,10 @@ variables: enum: - "HPC_OL7" - "HPC_OL8" - - "GPU" + - "GPU_OL7" + - "GPU_OL8" default: "HPC_OL7" visible: and: - ${use_marketplace_image_login} - - not: - - ${use_old_marketplace_image_login} - - not: - - ${use_standard_image_login} - - old_marketplace_listing_login: - type: enum - title: "Image version" - description: "Marketplace listing to use" - required: true - enum: - - "1. Oracle Linux 7.9 OFED 5.3-1.0.0.1 RHCK 20210607" - - "2. Oracle Linux 7.8 OFED 5.0-1.0.0.0 UEK 20200826" - - "3. Oracle Linux 7.7 OFED 4.4-2.0.7.0 UEK 20200229" - - "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" - default: "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" - visible: - and: - - ${use_marketplace_image_login} - - ${use_old_marketplace_image_login} - - not: - - ${use_standard_image_login} + - ${login_node} diff --git a/scripts/ib_write_bw.sh b/scripts/ib_write_bw.sh index 4af85b6e..58138096 100644 --- a/scripts/ib_write_bw.sh +++ b/scripts/ib_write_bw.sh @@ -20,7 +20,7 @@ dis_help() echo echo "Logs are stored at /tmp/logs" echo - echo "e.g., sh ./ib_write_bw.sh -s compute-permanent-node-1 -n compute-permanent-node-2 -c y -g 2 + echo "e.g., sh ./ib_write_bw.sh -s compute-permanent-node-1 -n compute-permanent-node-2 -c y -g 2" echo echo "Supported shapes: BM.GPU.B4.8,BM.GPU.A100-v2.8,BM.GPU4.8" echo @@ -51,13 +51,6 @@ do esac done -#Set variables -cuda_path=`ssh $server /usr/sbin/alternatives --list|grep cuda | awk -F" " '{print $3}'|tail -1`/targets/x86_64-linux/include/cuda.h -server_ip=`grep $server /etc/hosts |grep -v rdma|awk '{print $1}'` -logdir=/tmp/logs/ib_bw/`date +%F-%H` -outdir=/tmp/ib_bw/ -gpu_count=`ssh $server nvidia-smi -L |wc -l` - #Check node shape shape=`ssh $server 'curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ | jq .shape'` @@ -76,6 +69,13 @@ else exit fi +#Set variables +cuda_path=`ssh $server /usr/sbin/alternatives --list|grep cuda | awk -F" " '{print $3}'|tail -1`/targets/x86_64-linux/include/cuda.h +server_ip=`grep $server /etc/hosts |grep -v rdma|awk '{print $1}'` +logdir=/tmp/logs/ib_bw/`date +%F-%H` +outdir=/tmp/ib_bw/ +gpu_count=`ssh $server nvidia-smi -L |wc -l` + #check cuda installation ssh -q $server [[ -f $cuda_path ]] && echo " " || echo "Please check cuda installation; exit 1"; diff --git a/slurm_ha.tf b/slurm_ha.tf index 896b5d28..bc3d04cd 100644 --- a/slurm_ha.tf +++ b/slurm_ha.tf @@ -1,20 +1,11 @@ -resource "oci_core_volume" "backup_volume" { - count = var.bastion_block && var.slurm_ha ? 1 : 0 - availability_domain = var.bastion_ad - compartment_id = var.targetCompartment - display_name = "${local.cluster_name}-backup-volume" - size_in_gbs = var.bastion_block_volume_size - vpus_per_gb = split(".", var.bastion_block_volume_performance)[0] -} - - resource "oci_core_volume_attachment" "backup_volume_attachment" { count = var.bastion_block && var.slurm_ha ? 1 : 0 attachment_type = "iscsi" - volume_id = oci_core_volume.backup_volume[0].id + volume_id = oci_core_volume.bastion_volume[0].id instance_id = oci_core_instance.backup[0].id display_name = "${local.cluster_name}-backup-volume-attachment" device = "/dev/oracleoci/oraclevdb" + is_shareable = true } resource "oci_core_instance" "backup" { @@ -60,7 +51,7 @@ resource "oci_core_instance" "backup" { resource "null_resource" "backup" { count = var.slurm_ha ? 1 : 0 - depends_on = [oci_core_instance.backup, oci_core_volume_attachment.backup_volume_attachment ] + depends_on = [oci_core_instance.backup] triggers = { backup = oci_core_instance.backup[0].id } @@ -186,7 +177,7 @@ resource "null_resource" "backup" { } resource "null_resource" "cluster_backup" { count = var.slurm_ha ? 1 : 0 - depends_on = [null_resource.backup, oci_core_compute_cluster.compute_cluster, oci_core_cluster_network.cluster_network, oci_core_instance.backup, oci_core_volume_attachment.backup_volume_attachment ] + depends_on = [null_resource.backup, oci_core_compute_cluster.compute_cluster, oci_core_cluster_network.cluster_network, oci_core_instance.backup ] triggers = { cluster_instances = join(", ", local.cluster_instances_names) } @@ -221,14 +212,17 @@ resource "null_resource" "cluster_backup" { log_vol = var.log_vol, redundancy = var.redundancy, cluster_network = var.cluster_network, + use_compute_agent = var.use_compute_agent, slurm = var.slurm, slurm_nfs_path = var.slurm_nfs ? var.nfs_source_path : var.cluster_nfs_path, rack_aware = var.rack_aware, spack = var.spack, ldap = var.ldap, bastion_block = var.bastion_block, + login_block = var.login_block, scratch_nfs_type = local.scratch_nfs_type, bastion_mount_ip = local.bastion_mount_ip, + login_mount_ip = local.login_mount_ip, cluster_mount_ip = local.mount_ip, autoscaling = var.autoscaling, cluster_name = local.cluster_name, @@ -299,11 +293,11 @@ resource "null_resource" "cluster_backup" { provisioner "file" { content = templatefile("${path.module}/queues.conf", { cluster_network = var.cluster_network, + use_compute_agent = var.use_compute_agent, compute_cluster = var.compute_cluster, marketplace_listing = var.marketplace_listing, image = local.image_ocid, use_marketplace_image = var.use_marketplace_image, - use_old_marketplace_image = var.use_old_marketplace_image, boot_volume_size = var.boot_volume_size, shape = var.cluster_network ? var.cluster_network_shape : var.instance_pool_shape, region = var.region, @@ -351,8 +345,10 @@ resource "null_resource" "cluster_backup" { spack = var.spack, ldap = var.ldap, bastion_block = var.bastion_block, + login_block = var.login_block, scratch_nfs_type = local.scratch_nfs_type, bastion_mount_ip = local.bastion_mount_ip, + login_mount_ip = local.login_mount_ip, cluster_mount_ip = local.mount_ip, scratch_nfs_type_cluster = var.scratch_nfs_type_cluster, scratch_nfs_type_pool = var.scratch_nfs_type_pool, @@ -365,8 +361,6 @@ resource "null_resource" "cluster_backup" { ssh_cidr = var.ssh_cidr, use_cluster_nfs = var.use_cluster_nfs, cluster_nfs_path = var.cluster_nfs_path, - bastion_block = var.bastion_block, - bastion_mount_ip = local.bastion_mount_ip, home_nfs = var.home_nfs, create_fss = var.create_fss, home_fss = var.home_fss, @@ -392,7 +386,8 @@ resource "null_resource" "cluster_backup" { private_deployment = var.private_deployment, bastion_username = var.bastion_username, compute_username = var.compute_username, - use_multiple_ads = var.use_multiple_ads + use_multiple_ads = var.use_multiple_ads, + use_compute_agent = var.use_compute_agent }) destination = "/opt/oci-hpc/conf/variables.tf" diff --git a/variables.tf b/variables.tf index b49f100d..20c4d9ca 100755 --- a/variables.tf +++ b/variables.tf @@ -16,8 +16,6 @@ variable "cluster_name" { default = "" } variable "bastion_ad" {} variable "bastion_shape" { default = "VM.Standard2.4" } variable "bastion_object_storage_par" { default = true } -variable "use_standard_image" { default= true } -variable "use_standard_image_login" { default= true } variable "custom_bastion_image" { type = string default = "image.ocid" @@ -37,9 +35,9 @@ variable "instance_pool_shape" { default = "VM.Standard2.4" } variable "node_count" { default = 2 } variable "boot_volume_size" { default = 50 } variable "use_marketplace_image" { default = true} -variable "use_old_marketplace_image" { default = false} variable "image" { default = "ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa" } variable "image_ocid" { default = "ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa" } +variable "use_compute_agent" { default = false } variable "unsupported_bastion_image" { default = "" } variable "unsupported_login_image" { default = "" } variable "use_cluster_nfs" { default = true} @@ -85,10 +83,7 @@ variable "privilege_group_name" { default = "privilege" } variable "marketplace_listing" { default = "HPC_OL7" -} -variable "old_marketplace_listing" { - default = "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" -} +} variable "marketplace_version_id" { type = map(string) default = { @@ -96,20 +91,16 @@ variable "marketplace_version_id" { "2" = "OL7.8-OFED5.0-1.0.0.0-UEK-20200826" "3" = "OL7.7-OFED-4.4-2.0.7.0-UEK-20200229" "4" = "OL7.9-OFED5.0-2.1.8.0-RHCK-20210709" - "HPC_OL7" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.6.8.1-2023.05.18-0" - "HPC_OL8" = "OracleLinux-8-RHCK-OFED-5.4-3.6.8.1-2023.05.18-0" - "HPC_OL7_old" = "OL7.9-RHCK-3.10.0-OFED-5.4-3.4.0-1" - "HPC_OL8_old" = "OracleLinux-8-RHCK-OFED-5.4-3.5.8.0-2022.11.15-0" - "GPU_old" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.4.0.0-GPU-510-2022.09.23-1" - "GPU" = "OracleLinux-7-RHCK-3.10.0-OFED-5.4-3.6.8.1-GPU-515-2023.05.18-0" + "HPC_OL7" = "OracleLinux-7-OCA-RHCK-OFED-5.8-3.0.7.0-2024.01.02-0" + "HPC_OL8" = "OracleLinux-8-OCA-RHCK-OFED-5.8-3.0.7.0-2024.01.02-1" + "GPU_OL7" = "OracleLinux-7-OCA-RHCK-OFED-5.8-3.0.7.0-GPU-535-2024.01.02-0" + "GPU_OL8" = "OracleLinux-8-OCA-RHCK-OFED-5.8-3.0.7.0-GPU-535-2024.01.02-1" } } # To find the Appcatalog OCID, run # oci compute pic listing list --display-name "Oracle Linux 7 - HPC Cluster Networking Image" -variable "old_marketplace_listing_id" { - default = "ocid1.appcataloglisting.oc1..aaaaaaaahzcnanlki5vonyaeoiajjisejikzczygqqwheifymjqx3ft4iowa" -} + variable "marketplace_listing_id_HPC" { default = "ocid1.appcataloglisting.oc1..aaaaaaaahz2xiwfcsbebmqg7sp6lhdt6r2vsjro5jfukkl5cntlqvfhkbzaq" } @@ -196,6 +187,10 @@ variable "unsupported_bastion" { type=bool default = false } +variable "use_marketplace_image_bastion" { + type=bool + default = true +} variable "unsupported_login" { type=bool default = false @@ -251,13 +246,11 @@ variable "log_vol" { default = false } variable "redundancy" { default = true } variable "use_marketplace_image_login" { default = true} -variable "use_old_marketplace_image_login" { default = false} variable "marketplace_listing_login" { default = "HPC_OL7" } - -variable "old_marketplace_listing_login" { - default = "4. Oracle Linux 7.9 OFED 5.0-2.1.8.0 RHCK 20210709" -} +variable "marketplace_listing_bastion" { + default = "HPC_OL7" +} \ No newline at end of file