Merge pull request #48 from data-platform-hq/feat/default-cluster-policy-owerride

feat: cluster policies override

Author: owlleg6

README.md

@@ -183,6 +183,7 @@ No modules.
 |------|------|
 | [azurerm_key_vault_access_policy.databricks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
 | [databricks_cluster.cluster](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster) | resource |
+| [databricks_cluster_policy.overrides](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster_policy) | resource |
 | [databricks_cluster_policy.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster_policy) | resource |
 | [databricks_entitlements.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/entitlements) | resource |
 | [databricks_group.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/group) | resource |
@@ -215,6 +216,7 @@ No modules.
 | <a name="input_clusters"></a> [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups | <pre>set(object({<br>    cluster_name                 = string<br>    spark_version                = optional(string, "13.3.x-scala2.12")<br>    spark_conf                   = optional(map(any), {})<br>    cluster_conf_passthrought    = optional(bool, false)<br>    spark_env_vars               = optional(map(any), {})<br>    data_security_mode           = optional(string, "USER_ISOLATION")<br>    node_type_id                 = optional(string, "Standard_D3_v2")<br>    autotermination_minutes      = optional(number, 30)<br>    min_workers                  = optional(number, 1)<br>    max_workers                  = optional(number, 2)<br>    availability                 = optional(string, "ON_DEMAND_AZURE")<br>    first_on_demand              = optional(number, 0)<br>    spot_bid_max_price           = optional(number, 1)<br>    cluster_log_conf_destination = optional(string, null)<br>    init_scripts_workspace       = optional(set(string), [])<br>    init_scripts_volumes         = optional(set(string), [])<br>    init_scripts_dbfs            = optional(set(string), [])<br>    init_scripts_abfss           = optional(set(string), [])<br>    single_user_name             = optional(string, null)<br>    single_node_enable           = optional(bool, false)<br>    custom_tags                  = optional(map(string), {})<br>    permissions = optional(set(object({<br>      group_name       = string<br>      permission_level = string<br>    })), [])<br>    pypi_library_repository = optional(set(string), [])<br>    maven_library_repository = optional(set(object({<br>      coordinates = string<br>      exclusions  = set(string)<br>    })), [])<br>  }))</pre> | `[]` | no |
 | <a name="input_create_databricks_access_policy_to_key_vault"></a> [create\_databricks\_access\_policy\_to\_key\_vault](#input\_create\_databricks\_access\_policy\_to\_key\_vault) | Boolean flag to enable creation of Key Vault Access Policy for Databricks Global Service Principal. | `bool` | `true` | no |
 | <a name="input_custom_cluster_policies"></a> [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups<br>name - name of custom cluster policy to create<br>can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;<br>definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; | <pre>list(object({<br>    name       = string<br>    can_use    = list(string)<br>    definition = any<br>  }))</pre> | <pre>[<br>  {<br>    "can_use": null,<br>    "definition": null,<br>    "name": null<br>  }<br>]</pre> | no |
+| <a name="input_default_cluster_policies_override"></a> [default\_cluster\_policies\_override](#input\_default\_cluster\_policies\_override) | Provides an ability to override default cluster policy<br>name - name of cluster policy to override<br>family\_id - family id of corresponding policy<br>definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; | <pre>list(object({<br>    name       = string<br>    family_id  = string<br>    definition = any<br>  }))</pre> | <pre>[<br>  {<br>    "definition": null,<br>    "family_id": null,<br>    "name": null<br>  }<br>]</pre> | no |
 | <a name="input_global_databricks_sp_object_id"></a> [global\_databricks\_sp\_object\_id](#input\_global\_databricks\_sp\_object\_id) | Global 'AzureDatabricks' SP object id. Used to create Key Vault Access Policy for Secret Scope | `string` | `"9b38785a-6e08-4087-a0c4-20634343f21f"` | no |
 | <a name="input_iam_account_groups"></a> [iam\_account\_groups](#input\_iam\_account\_groups) | List of objects with group name and entitlements for this group | <pre>list(object({<br>    group_name   = optional(string)<br>    entitlements = optional(list(string))<br>  }))</pre> | `[]` | no |
 | <a name="input_iam_workspace_groups"></a> [iam\_workspace\_groups](#input\_iam\_workspace\_groups) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. | <pre>map(object({<br>    user              = optional(list(string))<br>    service_principal = optional(list(string))<br>    entitlements      = optional(list(string))<br>  }))</pre> | `{}` | no |

cluster.tf

@@ -115,3 +115,14 @@ resource "databricks_cluster_policy" "this" {
   name       = each.key
   definition = jsonencode(each.value)
 }
+
+resource "databricks_cluster_policy" "overrides" {
+  for_each = {
+    for param in var.default_cluster_policies_override : (param.name) => param
+    if param.definition != null
+  }
+
+  policy_family_id                   = each.value.family_id
+  policy_family_definition_overrides = jsonencode(each.value.definition)
+  name                               = each.key
+}

variables.tf

@@ -245,3 +245,22 @@ variable "system_schemas_enabled" {
   description = "System Schemas only works with assigned Unity Catalog Metastore. Boolean flag to enabled this feature"
   default     = false
 }
+
+variable "default_cluster_policies_override" {
+  type = list(object({
+    name       = string
+    family_id  = string
+    definition = any
+  }))
+  description = <<-EOT
+Provides an ability to override default cluster policy
+name - name of cluster policy to override
+family_id - family id of corresponding policy
+definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value;
+EOT
+  default = [{
+    name       = null
+    family_id  = null
+    definition = null
+  }]
+}