feat: add support for policy bundles and metrics SA (terraform-google-modules#1529)

Author: apeabody

examples/simple_zonal_with_acm/README.md

@@ -1,6 +1,6 @@
 # Simple Zonal Cluster
 
-This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/).
+This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/)'s [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview) and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) with the [Policy Essentials v2022 policy bundle](https://cloud.google.com/anthos-config-management/docs/how-to/using-policy-essentials-v2022).
 
 It incorporates the standard cluster module and the [ACM install module](../../modules/acm).
 
@@ -27,13 +27,19 @@ After applying the Terraform configuration, you can run the following commands t
  kubectl describe ns shipping-dev
  ```
 
+4. You can also use `kubectl` to view any policy violations on the cluster:
+
+ ```
+ kubectl get constraint -l policycontroller.gke.io/bundleName=policy-essentials-v2022 -o json | jq -cC '.items[]| [.metadata.name,.status.totalViolations]'
+ ```
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
-| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
+| project\_id | The project ID to host the cluster in | `string` | n/a | yes |
 | region | The region to host the cluster in | `string` | `"us-central1"` | no |
 | zone | The zone to host the cluster in | `string` | `"us-central1-a"` | no |
 

examples/simple_zonal_with_acm/acm.tf

@@ -25,4 +25,8 @@ module "acm" {
  policy_dir = "foo-corp"
 
  secret_type = "ssh"
+
+ policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"]
+
+ create_metrics_gcp_sa = true
 }

examples/simple_zonal_with_acm/variables.tf

@@ -16,15 +16,18 @@
 
 variable "project_id" {
  description = "The project ID to host the cluster in"
+ type = string
 }
 
 variable "cluster_name_suffix" {
  description = "A suffix to append to the default cluster name"
+ type = string
  default = ""
 }
 
 variable "region" {
  description = "The region to host the cluster in"
+ type = string
  default = "us-central1"
 }
 

examples/simple_zonal_with_acm/versions.tf

@@ -21,7 +21,12 @@ terraform {
  version = "~> 4.0"
  }
  kubernetes = {
- source = "hashicorp/kubernetes"
+ source = "hashicorp/kubernetes"
+ version = "~> 2.10"
+ }
+ random = {
+ source = "hashicorp/random"
+ version = ">= 2.1"
  }
  }
  required_version = ">= 0.13"

modules/acm/README.md

@@ -67,7 +67,9 @@ data "google_client_config" "default" {}
 | cluster\_membership\_id | The cluster membership ID. If unset, one will be autogenerated. | `string` | `""` | no |
 | cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes |
 | configmanagement\_version | Version of ACM. | `string` | `""` | no |
+| create\_metrics\_gcp\_sa | Create a Google service account for ACM metrics writing | `bool` | `false` | no |
 | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no |
+| enable\_config\_sync | Whether to enable the ACM Config Sync on the cluster | `bool` | `true` | no |
 | enable\_fleet\_feature | Whether to enable the ACM feature on the fleet. | `bool` | `true` | no |
 | enable\_fleet\_registration | Whether to create a new membership. | `bool` | `true` | no |
 | enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no |
@@ -77,19 +79,22 @@ data "google_client_config" "default" {}
 | https\_proxy | URL for the HTTPS proxy to be used when communicating with the Git repo. | `string` | `null` | no |
 | install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no |
 | location | GCP location used to reach cluster. | `string` | n/a | yes |
+| metrics\_gcp\_sa\_name | The name of the Google service account for ACM metrics writing | `string` | `"acm-metrics-writer"` | no |
+| policy\_bundles | A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster. | `list(string)` | `[]` | no |
 | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no |
 | project\_id | GCP project\_id used to reach cluster. | `string` | n/a | yes |
 | secret\_type | git authentication secret type, is passed through to ConfigManagement spec.git.secretType. Overriden to value 'ssh' if `create_ssh_key` is true | `string` | `"ssh"` | no |
 | source\_format | Configures a non-hierarchical repo if set to 'unstructured'. Uses [ACM defaults](https://cloud.google.com/anthos-config-management/docs/how-to/installing#configuring-config-management-operator) when unset. | `string` | `""` | no |
 | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |
 | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no |
-| sync\_repo | ACM Git repo address | `string` | n/a | yes |
+| sync\_repo | ACM Git repo address | `string` | `""` | no |
 | sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
+| acm\_metrics\_writer\_sa | The ACM metrics writer Service Account |
 | configmanagement\_version | Version of ACM installed. |
 | git\_creds\_public | Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository. |
 | wait | An output to use when you want to depend on cmd finishing |

modules/acm/creds.tf

@@ -14,6 +14,11 @@
  * limitations under the License.
  */
 
+locals {
+ # GCP service account ids must be <= 30 chars matching regex ^[a-z](?:[-a-z0-9]{4,28}[a-z0-9])$
+ service_account_name = trimsuffix(substr(var.metrics_gcp_sa_name, 0, 30), "-")
+}
+
 resource "tls_private_key" "k8sop_creds" {
  count = var.create_ssh_key ? 1 : 0
  algorithm = "RSA"
@@ -22,10 +27,92 @@ resource "tls_private_key" "k8sop_creds" {
 
 # Wait for the ACM operator to create the namespace
 resource "time_sleep" "wait_acm" {
- count = (var.create_ssh_key == true || var.ssh_auth_key != null) ? 1 : 0
+ count = (var.create_ssh_key == true || var.ssh_auth_key != null || var.enable_policy_controller || var.enable_config_sync) ? 1 : 0
+ depends_on = [google_gke_hub_feature_membership.main]
+
+ create_duration = "300s"
+}
+
+resource "google_service_account_iam_binding" "config-management-monitoring-iam" {
+ count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0
+ service_account_id = google_service_account.acm_metrics_writer_sa[0].name
+ role = "roles/iam.workloadIdentityUser"
+
+ members = ["serviceAccount:${var.project_id}.svc.id.goog[config-management-monitoring/default]"]
+
  depends_on = [google_gke_hub_feature_membership.main]
+}
+
+resource "google_service_account_iam_binding" "gatekeeper-system-iam" {
+ count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
+ service_account_id = google_service_account.acm_metrics_writer_sa[0].name
+ role = "roles/iam.workloadIdentityUser"
+
+ members = ["serviceAccount:${var.project_id}.svc.id.goog[gatekeeper-system/gatekeeper-admin]"]
+
+ depends_on = [google_gke_hub_feature_membership.main]
+}
+
+module "annotate-sa-config-management-monitoring" {
+ source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+ version = "~> 3.1"
+
+ count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0
+ skip_download = true
+ cluster_name = var.cluster_name
+ cluster_location = var.location
+ project_id = var.project_id
+
+ kubectl_create_command = "kubectl annotate --overwrite sa -n config-management-monitoring default iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}"
+ kubectl_destroy_command = "kubectl annotate sa -n config-management-monitoring default iam.gke.io/gcp-service-account-"
+
+ module_depends_on = time_sleep.wait_acm
+}
+
+module "annotate-sa-gatekeeper-system" {
+ source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+ version = "~> 3.1"
+
+ count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
+ skip_download = true
+ cluster_name = var.cluster_name
+ cluster_location = var.location
+ project_id = var.project_id
+
+ kubectl_create_command = "kubectl annotate --overwrite sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}"
+ kubectl_destroy_command = "kubectl annotate sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account-"
+
+ module_depends_on = time_sleep.wait_acm
+}
+
+module "annotate-sa-gatekeeper-system-restart" {
+ source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+ version = "~> 3.1"
+
+ count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
+ skip_download = true
+ cluster_name = var.cluster_name
+ cluster_location = var.location
+ project_id = var.project_id
+
+ kubectl_create_command = "kubectl rollout restart deployment gatekeeper-controller-manager -n gatekeeper-system"
+ kubectl_destroy_command = ""
+
+ module_depends_on = module.annotate-sa-gatekeeper-system
+}
+
+resource "google_service_account" "acm_metrics_writer_sa" {
+ count = var.create_metrics_gcp_sa ? 1 : 0
+
+ display_name = "ACM Metrics Writer SA"
+ account_id = local.service_account_name
+ project = var.project_id
+}
 
- create_duration = "60s"
+resource "google_project_iam_member" "acm_metrics_writer_sa_role" {
+ project = var.project_id
+ role = "roles/monitoring.metricWriter"
+ member = "serviceAccount:${google_service_account.acm_metrics_writer_sa[0].email}"
 }
 
 resource "kubernetes_secret_v1" "creds" {
@@ -38,6 +125,6 @@ resource "kubernetes_secret_v1" "creds" {
  }
 
  data = {
- "${local.k8sop_creds_secret_key}" = local.private_key
+ (local.k8sop_creds_secret_key) = local.private_key
  }
 }

modules/acm/feature.tf

@@ -38,16 +38,20 @@ resource "google_gke_hub_feature_membership" "main" {
  configmanagement {
  version = var.configmanagement_version
 
- config_sync {
- source_format = var.source_format != "" ? var.source_format : null
+ dynamic "config_sync" {
+ for_each = var.enable_config_sync ? [{ enabled = true }] : []
 
- git {
- sync_repo = var.sync_repo
- policy_dir = var.policy_dir != "" ? var.policy_dir : null
- sync_branch = var.sync_branch != "" ? var.sync_branch : null
- sync_rev = var.sync_revision != "" ? var.sync_revision : null
- secret_type = var.secret_type
- https_proxy = var.https_proxy
+ content {
+ source_format = var.source_format != "" ? var.source_format : null
+
+ git {
+ sync_repo = var.sync_repo
+ policy_dir = var.policy_dir != "" ? var.policy_dir : null
+ sync_branch = var.sync_branch != "" ? var.sync_branch : null
+ sync_rev = var.sync_revision != "" ? var.sync_revision : null
+ secret_type = var.secret_type
+ https_proxy = var.https_proxy
+ }
  }
  }
 

modules/acm/outputs.tf

@@ -16,7 +16,7 @@
 
 output "git_creds_public" {
  description = "Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository."
- value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds.*.public_key_openssh...) : null
+ value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds[*].public_key_openssh...) : null
 }
 
 output "configmanagement_version" {
@@ -31,3 +31,8 @@ output "wait" {
  google_gke_hub_feature_membership.main
  ]
 }
+
+output "acm_metrics_writer_sa" {
+ description = "The ACM metrics writer Service Account"
+ value = google_service_account.acm_metrics_writer_sa[0].email
+}

modules/acm/policy_bundles.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "policy_bundles" {
+ source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+ version = "~> 3.1"
+
+ for_each = toset(var.policy_bundles)
+ project_id = var.project_id
+ cluster_name = var.cluster_name
+ cluster_location = var.location
+ kubectl_create_command = "kubectl apply -k ${each.key}"
+ kubectl_destroy_command = "kubectl delete -k ${each.key}"
+
+ module_depends_on = [time_sleep.wait_acm]
+}

modules/acm/variables.tf

@@ -57,6 +57,7 @@ variable "configmanagement_version" {
 variable "sync_repo" {
  description = "ACM Git repo address"
  type = string
+ default = ""
 }
 
 variable "sync_branch" {
@@ -108,6 +109,12 @@ variable "ssh_auth_key" {
  default = null
 }
 
+variable "enable_config_sync" {
+ description = "Whether to enable the ACM Config Sync on the cluster"
+ type = bool
+ default = true
+}
+
 # Policy Controller config
 variable "enable_policy_controller" {
  description = "Whether to enable the ACM Policy Controller on the cluster"
@@ -139,3 +146,21 @@ variable "enable_referential_rules" {
  type = bool
  default = true
 }
+
+variable "policy_bundles" {
+ description = "A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster."
+ type = list(string)
+ default = []
+}
+
+variable "create_metrics_gcp_sa" {
+ description = "Create a Google service account for ACM metrics writing"
+ type = bool
+ default = false
+}
+
+variable "metrics_gcp_sa_name" {
+ description = "The name of the Google service account for ACM metrics writing"
+ type = string
+ default = "acm-metrics-writer"
+}