feat(TPG>=6.39)!: Fleet app operator permissions custom roles (#2377)

Co-authored-by: Andrew Peabody <andrewpeabody@google.com>

Author: tonybayvas

examples/simple_fleet_app_operator_permissions/main.tf

@@ -15,9 +15,11 @@
  */
 
 locals {
- app_operator_id = "app-operator-id"
- app_operator_team = "app-operator-team"
- app_operator_role = "VIEW"
+ app_operator_id = "app-operator-id"
+ app_operator_team = "app-operator-team"
+ app_operator_role = "VIEW"
+ custom_app_operator_id = "custom-app-operator-id"
+ custom_app_operator_role = "my-custom-role"
 }
 
 # Create a Service Account, which can be used as an app operator.
@@ -27,12 +29,31 @@ resource "google_service_account" "service_account" {
  display_name = "Test App Operator Service Account"
 }
 
+# Create another Service Account, which can be used as a custom role app operator.
+resource "google_service_account" "custom_service_account" {
+ project = var.fleet_project_id
+ account_id = local.custom_app_operator_id
+ display_name = "Test App Operator Custom Role Service Account"
+}
+
 # Create a Fleet Scope for the app operator's team.
 resource "google_gke_hub_scope" "scope" {
  project = var.fleet_project_id
  scope_id = local.app_operator_team
 }
 
+# Allowlist custom roles for usage in Scope RBAC
+resource "google_gke_hub_feature" "rbacrolebindingactuation" {
+ name = "rbacrolebindingactuation"
+ location = "global"
+ spec {
+ rbacrolebindingactuation {
+ allowed_custom_roles = [local.custom_app_operator_role]
+ }
+ }
+ project = var.fleet_project_id
+}
+
 # Grant permissions to the app operator to work with the Fleet Scope.
 module "permissions" {
  source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
@@ -48,3 +69,19 @@ module "permissions" {
  ]
 }
 
+# Grant custom role permissions to the app operator to work with the Fleet Scope.
+module "custom_permissions" {
+ source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
+ version = "~> 37.0"
+
+ fleet_project_id = var.fleet_project_id
+ scope_id = google_gke_hub_scope.scope.scope_id
+ users = ["${local.custom_app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"]
+ custom_role = local.custom_app_operator_role
+
+ depends_on = [
+ google_service_account.custom_service_account,
+ google_gke_hub_feature.rbacrolebindingactuation,
+ ]
+}
+

examples/simple_fleet_app_operator_permissions/versions.tf

@@ -20,11 +20,11 @@ terraform {
  required_providers {
  google = {
  source = "hashicorp/google"
- version = ">= 4.81.0"
+ version = ">= 6.39.0"
  }
  google-beta = {
  source = "hashicorp/google-beta"
- version = ">= 4.81.0"
+ version = ">= 6.39.0"
  }
  }
 }

modules/fleet-app-operator-permissions/README.md

@@ -1,4 +1,4 @@
-# Terrafrom Module for Fleet App Operator Permissions
+# Terraform Module for Fleet App Operator Permissions
 
 This module bundles different permissions (IAM and RBAC Role Bindings) required for [Fleet team management](https://cloud.google.com/kubernetes-engine/fleet-management/docs/team-management). A platform admin can use this module to set up permissions for an app operator (user or group) in a team--including usage of Fleet Scopes, Connect Gateway, logging, and metrics--based on predefined roles (VIEW, EDIT, ADMIN).
 
@@ -14,6 +14,17 @@ module "fleet_app_operator_permissions" {
  groups = ["people@company.com"]
  role = "EDIT"
 }
+
+Example:
+module "fleet_app_operator_permissions" {
+ source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
+
+ fleet_project_id = "my-project-id"
+ scope_id = "frontend-team"
+ users = ["person1@company.com", "person2@company.com"]
+ groups = ["people@company.com"]
+ custom_role = "my-custom-role"
+}
 ```
 
 To deploy this config, run:
@@ -28,9 +39,10 @@ To deploy this config, run:
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| custom\_role | The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set | `string` | `null` | no |
 | fleet\_project\_id | The project to which the Fleet belongs. | `string` | n/a | yes |
 | groups | The list of app operator group principals, e.g., `people@google.com`, `principalSet://iam.googleapis.com/locations/global/workforcePools/my-pool/group/people`. | `list(string)` | `[]` | no |
-| role | The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). | `string` | n/a | yes |
+| role | The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set | `string` | `null` | no |
 | scope\_id | The scope for which IAM and RBAC role bindings are created. | `string` | n/a | yes |
 | users | The list of app operator user principals, e.g., `person@google.com`, `principal://iam.googleapis.com/locations/global/workforcePools/my-pool/subject/person`, `serviceAccount:my-service-account@my-project.iam.gserviceaccount.com`. | `list(string)` | `[]` | no |
 

modules/fleet-app-operator-permissions/main.tf

@@ -27,15 +27,17 @@ locals {
  ))]
 
  project_level_scope_role = {
- "VIEW" = "roles/gkehub.scopeViewerProjectLevel"
- "EDIT" = "roles/gkehub.scopeEditorProjectLevel"
- "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
+ "VIEW" = "roles/gkehub.scopeViewerProjectLevel"
+ "EDIT" = "roles/gkehub.scopeEditorProjectLevel"
+ "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
+ "CUSTOM" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
  }
 
  resource_level_scope_role = {
- "VIEW" = "roles/gkehub.scopeViewer"
- "EDIT" = "roles/gkehub.scopeEditor"
- "ADMIN" = "roles/gkehub.scopeAdmin"
+ "VIEW" = "roles/gkehub.scopeViewer"
+ "EDIT" = "roles/gkehub.scopeEditor"
+ "ADMIN" = "roles/gkehub.scopeAdmin"
+ "CUSTOM" = "roles/gkehub.scopeViewer" # Same as VIEW
  }
 }
 
@@ -54,15 +56,16 @@ resource "google_project_iam_member" "log_view_permissions" {
 resource "google_project_iam_member" "project_level_scope_permissions" {
  project = var.fleet_project_id
  for_each = toset(concat(local.user_principals, local.group_principals))
- role = local.project_level_scope_role[var.role]
+ role = (var.custom_role != null ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role])
  member = each.value
 }
 
-resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" {
+resource "google_gke_hub_scope_iam_member" "resource_level_scope_permissions" {
  project = var.fleet_project_id
+ for_each = toset(concat(local.user_principals, local.group_principals))
  scope_id = var.scope_id
- role = local.resource_level_scope_role[var.role]
- members = concat(local.user_principals, local.group_principals)
+ role = (var.custom_role != null ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role])
+ member = each.value
 }
 
 resource "random_id" "user_rand_suffix" {
@@ -77,6 +80,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings
  scope_id = var.scope_id
  user = each.key
  role {
+ # Setting both types of roles will return an error when creating the resource.
+ custom_role = var.custom_role
  predefined_role = var.role
  }
 }
@@ -93,6 +98,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding
  scope_id = var.scope_id
  group = each.key
  role {
+ # Setting both types of roles will return an error when creating the resource.
+ custom_role = var.custom_role
  predefined_role = var.role
  }
 }

modules/fleet-app-operator-permissions/metadata.display.yaml

@@ -20,14 +20,17 @@ metadata:
  config.kubernetes.io/local-config: "true"
 spec:
  info:
- title: Terrafrom Module for Fleet App Operator Permissions
+ title: Terraform Module for Fleet App Operator Permissions
  source:
  repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git
  sourceType: git
  dir: /modules/fleet-app-operator-permissions
  ui:
  input:
  variables:
+ custom_role:
+ name: custom_role
+ title: Custom Role
  fleet_project_id:
  name: fleet_project_id
  title: Fleet Project Id

modules/fleet-app-operator-permissions/metadata.yaml

@@ -20,7 +20,7 @@ metadata:
  config.kubernetes.io/local-config: "true"
 spec:
  info:
- title: Terrafrom Module for Fleet App Operator Permissions
+ title: Terraform Module for Fleet App Operator Permissions
  source:
  repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git
  sourceType: git
@@ -139,9 +139,11 @@ spec:
  varType: list(string)
  defaultValue: []
  - name: role
- description: The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`).
+ description: The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set
+ varType: string
+ - name: custom_role
+ description: The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set
  varType: string
- required: true
  outputs:
  - name: fleet_project_id
  description: The project to which the Fleet belongs.

modules/fleet-app-operator-permissions/variables.tf

@@ -37,11 +37,18 @@ variable "groups" {
 }
 
 variable "role" {
- description = "The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)."
+ description = "The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set"
  type = string
  validation {
- condition = contains(["VIEW", "EDIT", "ADMIN"], var.role)
- error_message = "Allowed values for role are VIEW, EDIT, or ADMIN."
+ condition = var.role == null || contains(["VIEW", "EDIT", "ADMIN"], var.role)
+ error_message = "Allowed values for role are VIEW, EDIT, ADMIN, or null."
  }
+ default = null
+}
+
+variable "custom_role" {
+ description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set"
+ type = string
+ default = null
 }
 

modules/fleet-app-operator-permissions/versions.tf

@@ -20,11 +20,11 @@ terraform {
  required_providers {
  google = {
  source = "hashicorp/google"
- version = ">= 4.81.0"
+ version = ">= 6.39.0"
  }
  google-beta = {
  source = "hashicorp/google-beta"
- version = ">= 4.81.0"
+ version = ">= 6.39.0"
  }
  random = {
  source = "hashicorp/random"

test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go

@@ -39,23 +39,37 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) {
 appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail)
 scopeLevelRole := "roles/gkehub.scopeViewer"
 projectLevelRole := "roles/gkehub.scopeViewerProjectLevel"
+customAppOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId)
+customAppOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", customAppOperatorEmail)
+customScopeLevelRole := "roles/gkehub.scopeViewer"
+customProjectLevelRole := "roles/gkehub.scopeEditorProjectLevel"
 logViewRole := "roles/logging.viewAccessor"
 logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId)
 logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId)
+filterFormat := "\"bindings.members:%s\""
+flattenOpt := "bindings[].members"
 
 scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String()
 assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings")
+assert.Equal(strings.Contains(scopeRrbList, customAppOperatorEmail), true, "custom app operator email should be in the list of Scope RBAC Role Bindings")
 
-scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String()
-assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy")
+scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal)).String()
 assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy")
 
-projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String()
-assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy")
+customScopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal)).String()
+assert.Equal(strings.Contains(customScopeIam, customScopeLevelRole), true, "custom app operator Scope role should be in the Scope IAM policy")
+
+projectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal), flattenOpt).String()
 assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy")
 assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy")
 assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy")
 assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy")
+
+customProjectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal), flattenOpt).String()
+assert.Equal(strings.Contains(customProjectIam, customProjectLevelRole), true, "custom app operator Scope role should be in the project IAM policy")
+assert.Equal(strings.Contains(customProjectIam, logViewRole), true, "custom app operator log view role should be in the project IAM policy")
+assert.Equal(strings.Contains(customProjectIam, logViewContainerBucket), true, "custom app operator log view container bucket should be in the project IAM policy")
+assert.Equal(strings.Contains(customProjectIam, logViewPodBucket), true, "custom app operator log view pod bucket should be in the project IAM policy")
 })
 
 appOppT.Test()