From 9ceee83463372d2ff61c7bfbd2beeecf0527a5c2 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Tue, 12 Sep 2023 11:18:15 -0400
Subject: [PATCH 1/3] Add `trailing_delete` var to `dandiset_bucket` module

---
 terraform/modules/dandiset_bucket/variables.tf | 7 +++++++
 terraform/sponsored_bucket.tf                  | 2 ++
 terraform/staging_bucket.tf                    | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/terraform/modules/dandiset_bucket/variables.tf b/terraform/modules/dandiset_bucket/variables.tf
index 0eaf7db..41915c5 100644
--- a/terraform/modules/dandiset_bucket/variables.tf
+++ b/terraform/modules/dandiset_bucket/variables.tf
@@ -34,3 +34,10 @@ variable "log_bucket_name" {
   type        = string
   description = "The name of the log bucket."
 }
+
+# TODO: this can be inferred from the "versioning" variable once we're ready
+# to deploy this to the production bucket as well.
+variable "trailing_delete" {
+  type = bool
+  description = "Whether or not trailing delete should be enabled on the bucket."
+}
diff --git a/terraform/sponsored_bucket.tf b/terraform/sponsored_bucket.tf
index 3a58675..a1d32ce 100644
--- a/terraform/sponsored_bucket.tf
+++ b/terraform/sponsored_bucket.tf
@@ -3,6 +3,7 @@ module "sponsored_dandiset_bucket" {
   bucket_name                           = "dandiarchive"
   public                                = true
   versioning                            = true
+  trailing_delete                       = false
   allow_cross_account_heroku_put_object = true
   heroku_user                           = data.aws_iam_user.api
   log_bucket_name                       = "dandiarchive-logs"
@@ -16,6 +17,7 @@ module "sponsored_embargo_bucket" {
   source          = "./modules/dandiset_bucket"
   bucket_name     = "dandiarchive-embargo"
   versioning      = false
+  trailing_delete = false
   heroku_user     = data.aws_iam_user.api
   log_bucket_name = "dandiarchive-embargo-logs"
   providers = {
diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf
index c0481d6..b6ceeea 100644
--- a/terraform/staging_bucket.tf
+++ b/terraform/staging_bucket.tf
@@ -3,6 +3,7 @@ module "staging_dandiset_bucket" {
   bucket_name             = "dandi-api-staging-dandisets"
   public                  = true
   versioning              = true
+  trailing_delete         = true
   allow_heroku_put_object = true
   heroku_user             = data.aws_iam_user.api_staging
   log_bucket_name         = "dandi-api-staging-dandiset-logs"
@@ -16,6 +17,7 @@ module "staging_embargo_bucket" {
   source          = "./modules/dandiset_bucket"
   bucket_name     = "dandi-api-staging-embargo-dandisets"
   versioning      = false
+  trailing_delete = false
   heroku_user     = data.aws_iam_user.api_staging
   log_bucket_name = "dandi-api-staging-embargo-dandisets-logs"
   providers = {

From e3eae5b1b1a2e84dcea1605521f8df1517138286 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Tue, 12 Sep 2023 11:23:29 -0400
Subject: [PATCH 2/3] Add bucket policy preventing object version deletion

For now, gate it behind the `trailing_delete` flag and only enable it on
staging.
---
 terraform/modules/dandiset_bucket/main.tf | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf
index f910f00..a5c6390 100644
--- a/terraform/modules/dandiset_bucket/main.tf
+++ b/terraform/modules/dandiset_bucket/main.tf
@@ -236,4 +236,27 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
       identifiers = [var.heroku_user.arn]
     }
   }
+
+  dynamic "statement" {
+    for_each = var.trailing_delete ? [1] : []
+
+    content {
+      sid = "PreventDeletionOfObjectVersions"
+
+      resources = [
+        "${aws_s3_bucket.dandiset_bucket.arn}/*"
+      ]
+
+      actions = [
+        "s3:DeleteObjectVersion",
+      ]
+
+      effect = "Deny"
+
+      principals {
+        identifiers = ["*"]
+        type        = "*"
+      }
+    }
+  }
 }

From 275a5170ecb28a83b031aa16cb5cea7402956e6d Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Mon, 18 Sep 2023 10:00:14 -0400
Subject: [PATCH 3/3] Add lifecycle policy to expire deleted objects after 30
 days

---
 terraform/modules/dandiset_bucket/main.tf | 30 +++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf
index a5c6390..c3948ff 100644
--- a/terraform/modules/dandiset_bucket/main.tf
+++ b/terraform/modules/dandiset_bucket/main.tf
@@ -260,3 +260,33 @@ data "aws_iam_policy_document" "dandiset_bucket_policy" {
     }
   }
 }
+
+
+# S3 lifecycle policy that permanently deletes objects with delete markers
+# after 30 days.
+resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
+  # Must have bucket versioning enabled first
+  depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
+
+  count = var.trailing_delete ? 1 : 0
+
+  bucket = aws_s3_bucket.dandiset_bucket.id
+
+  # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7
+  rule {
+    id = "ExpireOldDeleteMarkers"
+    filter {}
+
+    # Expire objects with delete markers after 30 days
+    noncurrent_version_expiration {
+      noncurrent_days = 30
+    }
+
+    # Also delete any delete markers associated with the expired object
+    expiration {
+      expired_object_delete_marker = true
+    }
+
+    status = "Enabled"
+  }
+}