Merge pull request #157 from dandi/implement-trailing-delete

dandi · Sep 25, 2023 · dd31149 · dd31149
2 parents b2a7076 + 275a517
commit dd31149
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 0 deletions.
diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf
@@ -236,4 +236,57 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
       identifiers = [var.heroku_user.arn]
     }
   }
+
+  dynamic "statement" {
+    for_each = var.trailing_delete ? [1] : []
+
+    content {
+      sid = "PreventDeletionOfObjectVersions"
+
+      resources = [
+        "${aws_s3_bucket.dandiset_bucket.arn}/*"
+      ]
+
+      actions = [
+        "s3:DeleteObjectVersion",
+      ]
+
+      effect = "Deny"
+
+      principals {
+        identifiers = ["*"]
+        type        = "*"
+      }
+    }
+  }
+}
+
+
+# S3 lifecycle policy that permanently deletes objects with delete markers
+# after 30 days.
+resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
+  # Must have bucket versioning enabled first
+  depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
+
+  count = var.trailing_delete ? 1 : 0
+
+  bucket = aws_s3_bucket.dandiset_bucket.id
+
+  # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7
+  rule {
+    id = "ExpireOldDeleteMarkers"
+    filter {}
+
+    # Expire objects with delete markers after 30 days
+    noncurrent_version_expiration {
+      noncurrent_days = 30
+    }
+
+    # Also delete any delete markers associated with the expired object
+    expiration {
+      expired_object_delete_marker = true
+    }
+
+    status = "Enabled"
+  }
 }
diff --git a/terraform/modules/dandiset_bucket/variables.tf b/terraform/modules/dandiset_bucket/variables.tf
@@ -34,3 +34,10 @@ variable "log_bucket_name" {
   type        = string
   description = "The name of the log bucket."
 }
+
+# TODO: this can be inferred from the "versioning" variable once we're ready
+# to deploy this to the production bucket as well.
+variable "trailing_delete" {
+  type = bool
+  description = "Whether or not trailing delete should be enabled on the bucket."
+}
diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf
@@ -3,6 +3,7 @@ module "sponsored_dandiset_bucket" {
   bucket_name                           = "dandiarchive"
   public                                = true
   versioning                            = true
+  trailing_delete                       = false
   allow_cross_account_heroku_put_object = true
   heroku_user                           = data.aws_iam_user.api
   log_bucket_name                       = "dandiarchive-logs"
@@ -16,6 +17,7 @@ module "sponsored_embargo_bucket" {
   source          = "./modules/dandiset_bucket"
   bucket_name     = "dandiarchive-embargo"
   versioning      = false
+  trailing_delete = false
   heroku_user     = data.aws_iam_user.api
   log_bucket_name = "dandiarchive-embargo-logs"
   providers = {

diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf
@@ -3,6 +3,7 @@ module "staging_dandiset_bucket" {
   bucket_name             = "dandi-api-staging-dandisets"
   public                  = true
   versioning              = true
+  trailing_delete         = true
   allow_heroku_put_object = true
   heroku_user             = data.aws_iam_user.api_staging
   log_bucket_name         = "dandi-api-staging-dandiset-logs"
@@ -16,6 +17,7 @@ module "staging_embargo_bucket" {
   source          = "./modules/dandiset_bucket"
   bucket_name     = "dandi-api-staging-embargo-dandisets"
   versioning      = false
+  trailing_delete = false
   heroku_user     = data.aws_iam_user.api_staging
   log_bucket_name = "dandi-api-staging-embargo-dandisets-logs"
   providers = {