diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 67f4f6e..84e1356 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -1,183 +1,183 @@ ---- -name: Generate Data Usage Report - -on: - pull_request: - branches: - - main - - -jobs: - generate-jobs-usage-report: - runs-on: ubuntu-latest - - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Launch EC2 Instance - id: launch_ec2 - run: | - INSTANCE_ID=$(aws ec2 run-instances \ - --image-id ami-088d38b423bff245f \ - --count 1 \ - --instance-type t3.micro \ - --key-name dandihub-gh-actions \ - --security-group-ids sg-0bf2dc1c2ff9c122e \ - --subnet-id subnet-0f544cca61ccd2804 \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=dandihub-gh-actions}]" \ - --query 'Instances[0].InstanceId' --output text) - - echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_ENV - - - name: Wait for EC2 to Initialize - run: | - aws ec2 wait instance-status-ok --instance-ids ${{ env.INSTANCE_ID }} - - - name: Allocate Elastic IP - id: allocate_eip - run: | - ALLOC_ID=$(aws ec2 allocate-address --query 'AllocationId' --output text) - echo "ALLOC_ID=${ALLOC_ID}" >> $GITHUB_ENV - - - name: Associate Elastic IP with EC2 Instance - id: associate_eip - run: | - EIP=$(aws ec2 associate-address \ - --instance-id ${{ env.INSTANCE_ID }} \ - --allocation-id ${{ env.ALLOC_ID }} \ - --query 'AssociationId' --output text) - echo "EIP=${EIP}" >> $GITHUB_ENV - - - name: Retrieve Elastic IP Address - id: get_ip - run: | - PUBLIC_IP=$(aws ec2 describe-addresses \ - --allocation-ids ${{ env.ALLOC_ID }} \ - --query 'Addresses[0].PublicIp' --output text) - echo "PUBLIC_IP=${PUBLIC_IP}" - echo "PUBLIC_IP=${PUBLIC_IP}" >> $GITHUB_ENV - - - name: Execute df Command on EC2 - uses: appleboy/ssh-action@v0.1.6 - with: - host: ${{ env.PUBLIC_IP }} - username: ec2-user - key: ${{ secrets.EC2_SSH_KEY }} - script: | - echo "Running df command on EC2 instance..." - df -h - echo "Command completed." - continue-on-error: true # Allow the workflow to continue even if this step fails - - - - name: Terminate EC2 Instance - run: | - aws ec2 terminate-instances --instance-ids ${{ env.INSTANCE_ID }} - aws ec2 wait instance-terminated --instance-ids ${{ env.INSTANCE_ID }} - continue-on-error: true # Allow the workflow to continue even if this step fails - - - name: Release Elastic IP - run: | - aws ec2 release-address --allocation-id ${{ env.ALLOC_ID }} - continue-on-error: true # Allow the workflow to continue even if this step fails - -# jobs: -# generate_data_usage_report: -# runs-on: ubuntu-latest +# --- +# name: Generate Data Usage Report # -# steps: +# on: +# pull_request: +# branches: +# - main # -# - name: Log in to DockerHub -# uses: docker/login-action@v2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} # -# - name: Build and push Docker image -# uses: docker/build-push-action@v3 -# with: -# context: . -# file: images/Dockerfile.dandihub_report_generator -# push: true -# tags: ${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest +# jobs: +# generate-jobs-usage-report: +# runs-on: ubuntu-latest # -# - name: Configure AWS credentials +# steps: +# - name: Configure AWS Credentials # uses: aws-actions/configure-aws-credentials@v3 # with: # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # aws-region: us-east-2 # -# - name: Assume ProvisioningRole +# - name: Launch EC2 Instance +# id: launch_ec2 # run: | -# CREDS=$(aws sts assume-role --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} --role-session-name "GitHubActionsSession") -# export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') -# export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') -# export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') +# INSTANCE_ID=$(aws ec2 run-instances \ +# --image-id ami-088d38b423bff245f \ +# --count 1 \ +# --instance-type t3.micro \ +# --key-name dandihub-gh-actions \ +# --security-group-ids sg-0bf2dc1c2ff9c122e \ +# --subnet-id subnet-0f544cca61ccd2804 \ +# --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=dandihub-gh-actions}]" \ +# --query 'Instances[0].InstanceId' --output text) +# +# echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_ENV # -# - name: Configure kubectl with AWS EKS +# - name: Wait for EC2 to Initialize # run: | -# aws eks update-kubeconfig --name eks-dandihub --region us-east-2 --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} +# aws ec2 wait instance-status-ok --instance-ids ${{ env.INSTANCE_ID }} # -# # TODO remove -# - name: Sanity check +# - name: Allocate Elastic IP +# id: allocate_eip # run: | -# kubectl get pods -n jupyterhub +# ALLOC_ID=$(aws ec2 allocate-address --query 'AllocationId' --output text) +# echo "ALLOC_ID=${ALLOC_ID}" >> $GITHUB_ENV # -# # - name: Deploy Hello World Pod -# # run: | -# # kubectl apply -f .github/manifests/hello-world-pod.yaml -# # -# # - name: Wait for Hello World Pod to complete -# # run: | -# # kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=300s # 5 minutes -# # continue-on-error: true # Allow the workflow to continue even if this step fails -# # -# # - name: Get Hello World Pod logs -# # run: | -# # kubectl logs hello-world-pod -# # if: ${{ success() }} # Only run this step if the previous step was successful -# # -# # - name: Delete Hello World Pod -# # run: | -# # kubectl delete pod hello-world-pod -# # if: ${{ always() }} # Always run this step, even if other steps fail -# # -# - name: Replace image placeholder in manifest +# - name: Associate Elastic IP with EC2 Instance +# id: associate_eip # run: | -# sed -i 's|IMAGE_PLACEHOLDER|'"${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest"'|' .github/manifests/disk-usage-report-job.yaml +# EIP=$(aws ec2 associate-address \ +# --instance-id ${{ env.INSTANCE_ID }} \ +# --allocation-id ${{ env.ALLOC_ID }} \ +# --query 'AssociationId' --output text) +# echo "EIP=${EIP}" >> $GITHUB_ENV # -# - name: Deploy Disk Usage Report Job +# - name: Retrieve Elastic IP Address +# id: get_ip # run: | -# kubectl apply -f .github/manifests/disk-usage-report-job.yaml +# PUBLIC_IP=$(aws ec2 describe-addresses \ +# --allocation-ids ${{ env.ALLOC_ID }} \ +# --query 'Addresses[0].PublicIp' --output text) +# echo "PUBLIC_IP=${PUBLIC_IP}" +# echo "PUBLIC_IP=${PUBLIC_IP}" >> $GITHUB_ENV +# +# - name: Execute df Command on EC2 +# uses: appleboy/ssh-action@v0.1.6 +# with: +# host: ${{ env.PUBLIC_IP }} +# username: ec2-user +# key: ${{ secrets.EC2_SSH_KEY }} +# script: | +# echo "Running df command on EC2 instance..." +# df -h +# echo "Command completed." +# continue-on-error: true # Allow the workflow to continue even if this step fails +# # -# # TODO should timeout be longer? -# - name: Wait for Disk Usage Report Job to complete +# - name: Terminate EC2 Instance # run: | -# kubectl wait --for=condition=complete job/disk-usage-report-job --timeout=360s -n jupyterhub -# continue-on-error: true +# aws ec2 terminate-instances --instance-ids ${{ env.INSTANCE_ID }} +# aws ec2 wait instance-terminated --instance-ids ${{ env.INSTANCE_ID }} +# continue-on-error: true # Allow the workflow to continue even if this step fails # -# # continue-on-error for previous steps so we delete the job -# - name: Delete Disk Usage Report Job +# - name: Release Elastic IP # run: | -# kubectl delete job disk-usage-report-job -n jupyterhub +# aws ec2 release-address --allocation-id ${{ env.ALLOC_ID }} +# continue-on-error: true # Allow the workflow to continue even if this step fails # -# # - name: Clone dandi-hub-usage-reports repository -# # run: | -# # git clone https://github.com/dandi/dandi-hub-usage-reports.git -# # -# # - name: Copy report file to repository, commit and push report -# # run: | -# # cd dandi-hub-usage-reports -# # DATE=$(date +'%Y-%m-%d') -# # mv ../du_report.json $DATE_du_report.json -# # git config --global user.name "GitHub Actions" -# # git config --global user.email "actions@github.com" -# # git add $DATE_du_report.json -# # git commit -m "Add disk usage report for $DATE" -# # git push https://${{ secrets.GITHUB_TOKEN }}@github.com/dandi/dandi-hub-usage-reports.git +# # jobs: +# # generate_data_usage_report: +# # runs-on: ubuntu-latest +# # +# # steps: +# # +# # - name: Log in to DockerHub +# # uses: docker/login-action@v2 +# # with: +# # username: ${{ secrets.DOCKERHUB_USERNAME }} +# # password: ${{ secrets.DOCKERHUB_TOKEN }} +# # +# # - name: Build and push Docker image +# # uses: docker/build-push-action@v3 +# # with: +# # context: . +# # file: images/Dockerfile.dandihub_report_generator +# # push: true +# # tags: ${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest +# # +# # - name: Configure AWS credentials +# # uses: aws-actions/configure-aws-credentials@v3 +# # with: +# # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# # aws-region: us-east-2 +# # +# # - name: Assume ProvisioningRole +# # run: | +# # CREDS=$(aws sts assume-role --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} --role-session-name "GitHubActionsSession") +# # export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') +# # export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') +# # export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') +# # +# # - name: Configure kubectl with AWS EKS +# # run: | +# # aws eks update-kubeconfig --name eks-dandihub --region us-east-2 --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} +# # +# # # TODO remove +# # - name: Sanity check +# # run: | +# # kubectl get pods -n jupyterhub +# # +# # # - name: Deploy Hello World Pod +# # # run: | +# # # kubectl apply -f .github/manifests/hello-world-pod.yaml +# # # +# # # - name: Wait for Hello World Pod to complete +# # # run: | +# # # kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=300s # 5 minutes +# # # continue-on-error: true # Allow the workflow to continue even if this step fails +# # # +# # # - name: Get Hello World Pod logs +# # # run: | +# # # kubectl logs hello-world-pod +# # # if: ${{ success() }} # Only run this step if the previous step was successful +# # # +# # # - name: Delete Hello World Pod +# # # run: | +# # # kubectl delete pod hello-world-pod +# # # if: ${{ always() }} # Always run this step, even if other steps fail +# # # +# # - name: Replace image placeholder in manifest +# # run: | +# # sed -i 's|IMAGE_PLACEHOLDER|'"${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest"'|' .github/manifests/disk-usage-report-job.yaml +# # +# # - name: Deploy Disk Usage Report Job +# # run: | +# # kubectl apply -f .github/manifests/disk-usage-report-job.yaml +# # +# # # TODO should timeout be longer? +# # - name: Wait for Disk Usage Report Job to complete +# # run: | +# # kubectl wait --for=condition=complete job/disk-usage-report-job --timeout=360s -n jupyterhub +# # continue-on-error: true +# # +# # # continue-on-error for previous steps so we delete the job +# # - name: Delete Disk Usage Report Job +# # run: | +# # kubectl delete job disk-usage-report-job -n jupyterhub +# # +# # # - name: Clone dandi-hub-usage-reports repository +# # # run: | +# # # git clone https://github.com/dandi/dandi-hub-usage-reports.git +# # # +# # # - name: Copy report file to repository, commit and push report +# # # run: | +# # # cd dandi-hub-usage-reports +# # # DATE=$(date +'%Y-%m-%d') +# # # mv ../du_report.json $DATE_du_report.json +# # # git config --global user.name "GitHub Actions" +# # # git config --global user.email "actions@github.com" +# # # git add $DATE_du_report.json +# # # git commit -m "Add disk usage report for $DATE" +# # # git push https://${{ secrets.GITHUB_TOKEN }}@github.com/dandi/dandi-hub-usage-reports.git