feat(TPG>=5.44.2)!: add standard cluster support for insecureKubeletReadonlyPortEnabled (terraform-google-modules#2082)

Author: wyardley

README.md

@@ -190,6 +190,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
@@ -319,6 +320,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional |

autogen/main/README.md

@@ -213,6 +213,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional |

autogen/main/cluster.tf.tmpl

@@ -661,7 +661,6 @@ resource "google_container_cluster" "primary" {
  }
  }
  }
- {% if beta_cluster %}
 
  node_pool_defaults {
  node_config_defaults {
@@ -675,15 +674,17 @@ resource "google_container_cluster" "primary" {
  }
  {% endif %}
  {% if autopilot_cluster != true %}
+ {% if beta_cluster %}
  gcfs_config {
  enabled = var.enable_gcfs
  }
  {% endif %}
+ insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
+ {% endif %}
  }
  }
- {% endif %}
- {% if beta_cluster %}
 
+ {% if beta_cluster %}
  depends_on = [google_project_iam_member.service_agent]
  {% endif %}
 }
@@ -1046,14 +1047,15 @@ resource "google_container_node_pool" "windows_pools" {
  dynamic "kubelet_config" {
  for_each = length(setintersection(
  keys(each.value),
- ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+ ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
  )) != 0 ? [1] : []
 
  content {
- cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
- cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
- cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
- pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
+ cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
+ cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
+ cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
+ insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+ pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
  }
  }
  {% if beta_cluster %}

autogen/main/variables.tf.tmpl

@@ -109,6 +109,12 @@ variable "service_external_ips" {
 }
 
 {% if autopilot_cluster != true %}
+variable "insecure_kubelet_readonly_port_enabled" {
+ type = bool
+ description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+ default = null
+}
+
 variable "datapath_provider" {
  type = string
  description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

autogen/main/versions.tf.tmpl

@@ -24,11 +24,11 @@ terraform {
  required_providers {
  google = {
  source = "hashicorp/google"
- version = ">= 5.40.0, < 7"
+ version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
  }
  google-beta = {
  source = "hashicorp/google-beta"
- version = ">= 5.40.0, < 7"
+ version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
  }
  kubernetes = {
  source = "hashicorp/kubernetes"
@@ -89,7 +89,7 @@ terraform {
  required_providers {
  google = {
  source = "hashicorp/google"
- version = ">= 5.40.0, < 7"
+ version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
  }
  kubernetes = {
  source = "hashicorp/kubernetes"

cluster.tf

@@ -500,6 +500,13 @@ resource "google_container_cluster" "primary" {
  }
  }
  }
+
+ node_pool_defaults {
+ node_config_defaults {
+ insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
+ }
+ }
+
 }
 /******************************************
  Create Container Cluster node pools
@@ -739,14 +746,15 @@ resource "google_container_node_pool" "pools" {
  dynamic "kubelet_config" {
  for_each = length(setintersection(
  keys(each.value),
- ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+ ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
  )) != 0 ? [1] : []
 
  content {
- cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
- cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
- cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
- pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
+ cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
+ cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
+ cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
+ insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+ pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
  }
  }
 
@@ -1029,14 +1037,15 @@ resource "google_container_node_pool" "windows_pools" {
  dynamic "kubelet_config" {
  for_each = length(setintersection(
  keys(each.value),
- ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+ ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
  )) != 0 ? [1] : []
 
  content {
- cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
- cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
- cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
- pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
+ cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
+ cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
+ cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
+ insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+ pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
  }
  }
 

examples/node_pool/main.tf

@@ -67,20 +67,21 @@ module "gke" {
  service_account = var.compute_engine_service_account
  },
  {
- name = "pool-03"
- machine_type = "n1-standard-2"
- node_locations = "${var.region}-b,${var.region}-c"
- autoscaling = false
- node_count = 2
- disk_type = "pd-standard"
- auto_upgrade = true
- service_account = var.compute_engine_service_account
- pod_range = "test"
- sandbox_enabled = true
- cpu_manager_policy = "static"
- cpu_cfs_quota = true
- local_ssd_ephemeral_count = 2
- pod_pids_limit = 4096
+ name = "pool-03"
+ machine_type = "n1-standard-2"
+ node_locations = "${var.region}-b,${var.region}-c"
+ autoscaling = false
+ node_count = 2
+ disk_type = "pd-standard"
+ auto_upgrade = true
+ service_account = var.compute_engine_service_account
+ pod_range = "test"
+ sandbox_enabled = true
+ cpu_manager_policy = "static"
+ cpu_cfs_quota = true
+ insecure_kubelet_readonly_port_enabled = "FALSE"
+ local_ssd_ephemeral_count = 2
+ pod_pids_limit = 4096
  },
  {
  name = "pool-04"

examples/node_pool_update_variant/main.tf

@@ -61,11 +61,12 @@ module "gke" {
 
  node_pools = [
  {
- name = "pool-01"
- min_count = 1
- max_count = 2
- service_account = var.compute_engine_service_account
- auto_upgrade = true
+ name = "pool-01"
+ min_count = 1
+ max_count = 2
+ service_account = var.compute_engine_service_account
+ auto_upgrade = true
+ insecure_kubelet_readonly_port_enabled = "FALSE"
  },
  {
  name = "pool-02"

modules/beta-private-cluster-update-variant/README.md

@@ -233,6 +233,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
@@ -377,6 +378,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional |

modules/beta-private-cluster-update-variant/cluster.tf

@@ -581,6 +581,7 @@ resource "google_container_cluster" "primary" {
  gcfs_config {
  enabled = var.enable_gcfs
  }
+ insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
  }
  }
 
@@ -915,14 +916,15 @@ resource "google_container_node_pool" "pools" {
  dynamic "kubelet_config" {
  for_each = length(setintersection(
  keys(each.value),
- ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+ ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
  )) != 0 ? [1] : []
 
  content {
- cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
- cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
- cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
- pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
+ cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
+ cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
+ cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
+ insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+ pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
  }
  }
 
@@ -1219,14 +1221,15 @@ resource "google_container_node_pool" "windows_pools" {
  dynamic "kubelet_config" {
  for_each = length(setintersection(
  keys(each.value),
- ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+ ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
  )) != 0 ? [1] : []
 
  content {
- cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
- cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
- cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
- pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
+ cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
+ cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
+ cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
+ insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+ pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
  }
  }