#98 Feature/bottlerocket nodepool (#107)

* #94 bumped provider versions

* #94 updated lock files

* #94 updated acm, eks and r53 modules versions

* #98 updated default eks cluster version to 1.20

* #98 fixed s3 vpc endpoint

* #98 added bottlerocket self-managed worker group

* #98 added bottlerocket worker group params

* #98 migrated to node groups

* #98 fixed br labels

* #98 changed affinity for node groups in all layer2 services

* #98 renamed node_pool to node_group, updated eks module, added force update parameter

* #98 renamed node_pool to node_group, updated eks module, added force update parameter

* #98 removed worker groups variables, updated k8s version to 1.21

* #98 moved workers additional policies into variable

* #98 added descriptions for variables and updated Readme

* #98 removed eks version from demo tfvars

* #98 fixed typo

* #98 added vpc endpoint routes to public subnets

* #98 removed unused template

Author: halfb00t

examples/echoserver-deployment.yaml

@@ -44,7 +44,7 @@ metadata:
     kubernetes.io/ingress.class: alb
     alb.ingress.kubernetes.io/scheme: internet-facing
     alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01
-    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:730809894724:certificate/aa3c4d2f-c661-49b8-b312-1f1cf1b7ef51
+    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:730809894724:certificate/fa029132-86ab-4342-96e2-8e1fd5c56c29
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
     alb.ingress.kubernetes.io/target-type: ip
     external-dns.alpha.kubernetes.io/hostname: echo.maddevs.org
@@ -59,4 +59,3 @@ spec:
             backend:
               serviceName: echoserver
               servicePort: 80
-

terraform/layer1-aws/README.md

@@ -3,14 +3,14 @@
 | Name | Version |
 |------|---------|
 | terraform | 0.15.1 |
-| aws | 3.38.0 |
-| kubernetes | 2.1.0 |
+| aws | 3.53.0 |
+| kubernetes | 2.4.1 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| aws | 3.38.0 |
+| aws | 3.53.0 |
 
 ## Inputs
 
@@ -25,15 +25,22 @@
 | domain\_name | Main public domain name | `any` | n/a | yes |
 | ecr\_repo\_retention\_count | number of images to store in ECR | `number` | `50` | no |
 | ecr\_repos | List of docker repositories | `list(any)` | <pre>[<br>  "demo"<br>]</pre> | no |
-| eks\_cluster\_version | Version of the EKS K8S cluster | `string` | `"1.18"` | no |
+| eks\_cluster\_enabled\_log\_types | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html). Possible values: api, audit, authenticator, controllerManager, scheduler | `list(string)` | <pre>[<br>  "audit"<br>]</pre> | no |
+| eks\_cluster\_encryption\_config\_enable | Enable or not encryption for k8s secrets with aws-kms | `bool` | `false` | no |
+| eks\_cluster\_log\_retention\_in\_days | Number of days to retain log events. Default retention - 90 days. | `number` | `90` | no |
+| eks\_cluster\_version | Version of the EKS K8S cluster | `string` | `"1.21"` | no |
 | eks\_map\_roles | Additional IAM roles to add to the aws-auth configmap. | <pre>list(object({<br>    rolearn  = string<br>    username = string<br>    groups   = list(string)<br>  }))</pre> | `[]` | no |
-| eks\_worker\_groups | EKS Worker groups configuration | `map` | <pre>{<br>  "ci": {<br>    "asg_desired_capacity": 0,<br>    "asg_max_size": 3,<br>    "asg_min_size": 0,<br>    "override_instance_types": [<br>      "t3.medium",<br>      "t3a.medium"<br>    ],<br>    "spot_instance_pools": 2<br>  },<br>  "ondemand": {<br>    "asg_desired_capacity": 1,<br>    "asg_max_size": 6,<br>    "instance_type": "t3a.medium"<br>  },<br>  "spot": {<br>    "asg_desired_capacity": 1,<br>    "asg_max_size": 5,<br>    "asg_min_size": 0,<br>    "override_instance_types": [<br>      "t3.medium",<br>      "t3a.medium"<br>    ],<br>    "spot_instance_pools": 2<br>  }<br>}</pre> | no |
+| eks\_workers\_additional\_policies | Additional IAM policy attached to EKS worker nodes | `list(any)` | <pre>[<br>  "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"<br>]</pre> | no |
 | eks\_write\_kubeconfig | Flag for eks module to write kubeconfig | `bool` | `false` | no |
 | environment | Env name in case workspace wasn't used | `string` | `"demo"` | no |
-| name | Project name, required to form unique resource names | `any` | n/a | yes |
+| name | Project name, required to create unique resource names | `any` | n/a | yes |
+| node\_group\_ci | Node group configuration | <pre>object({<br>    instance_types       = list(string)<br>    capacity_type        = string<br>    max_capacity         = number<br>    min_capacity         = number<br>    desired_capacity     = number<br>    force_update_version = bool<br>  })</pre> | <pre>{<br>  "capacity_type": "SPOT",<br>  "desired_capacity": 0,<br>  "force_update_version": true,<br>  "instance_types": [<br>    "t3a.medium",<br>    "t3.medium"<br>  ],<br>  "max_capacity": 5,<br>  "min_capacity": 0<br>}</pre> | no |
+| node\_group\_ondemand | Node group configuration | <pre>object({<br>    instance_types       = list(string)<br>    capacity_type        = string<br>    max_capacity         = number<br>    min_capacity         = number<br>    desired_capacity     = number<br>    force_update_version = bool<br>  })</pre> | <pre>{<br>  "capacity_type": "ON_DEMAND",<br>  "desired_capacity": 1,<br>  "force_update_version": true,<br>  "instance_types": [<br>    "t3a.medium"<br>  ],<br>  "max_capacity": 5,<br>  "min_capacity": 1<br>}</pre> | no |
+| node\_group\_spot | Node group configuration | <pre>object({<br>    instance_types       = list(string)<br>    capacity_type        = string<br>    max_capacity         = number<br>    min_capacity         = number<br>    desired_capacity     = number<br>    force_update_version = bool<br>  })</pre> | <pre>{<br>  "capacity_type": "SPOT",<br>  "desired_capacity": 1,<br>  "force_update_version": true,<br>  "instance_types": [<br>    "t3a.medium",<br>    "t3.medium"<br>  ],<br>  "max_capacity": 5,<br>  "min_capacity": 0<br>}</pre> | no |
 | region | Default infrastructure region | `string` | `"us-east-1"` | no |
 | short\_region | The abbreviated name of the region, required to form unique resource names | `map` | <pre>{<br>  "ap-east-1": "ape1",<br>  "ap-northeast-1": "apn1",<br>  "ap-northeast-2": "apn2",<br>  "ap-south-1": "aps1",<br>  "ap-southeast-1": "apse1",<br>  "ap-southeast-2": "apse2",<br>  "ca-central-1": "cac1",<br>  "cn-north-1": "cnn1",<br>  "cn-northwest-1": "cnnw1",<br>  "eu-central-1": "euc1",<br>  "eu-north-1": "eun1",<br>  "eu-west-1": "euw1",<br>  "eu-west-2": "euw2",<br>  "eu-west-3": "euw3",<br>  "sa-east-1": "sae1",<br>  "us-east-1": "use1",<br>  "us-east-2": "use2",<br>  "us-gov-east-1": "usge1",<br>  "us-gov-west-1": "usgw1",<br>  "us-west-1": "usw1",<br>  "us-west-2": "usw2"<br>}</pre> | no |
 | single\_nat\_gateway | Flag to create single nat gateway for all AZs | `bool` | `true` | no |
+| worker\_group\_bottlerocket | Bottlerocket worker group configuration | <pre>object({<br>    instance_types      = list(string)<br>    capacity_type       = string<br>    max_capacity        = number<br>    min_capacity        = number<br>    desired_capacity    = number<br>    spot_instance_pools = number<br>  })</pre> | <pre>{<br>  "capacity_type": "SPOT",<br>  "desired_capacity": 0,<br>  "instance_types": [<br>    "t3a.medium",<br>    "t3.medium"<br>  ],<br>  "max_capacity": 5,<br>  "min_capacity": 0,<br>  "spot_instance_pools": 2<br>}</pre> | no |
 | zone\_id | R53 zone id for public domain | `any` | `null` | no |
 
 ## Outputs

terraform/layer1-aws/aws-acm.tf

@@ -5,9 +5,9 @@ module "acm" {
   create_certificate = var.create_acm_certificate
 
   domain_name = local.domain_name
+  zone_id     = local.zone_id
   subject_alternative_names = [
   "*.${local.domain_name}"]
-  zone_id = local.zone_id
 
   tags = local.tags
 }

terraform/layer1-aws/aws-eks-bottlerocket.tf

@@ -0,0 +1,8 @@
+data "aws_ami" "bottlerocket_ami" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["bottlerocket-aws-k8s-${var.eks_cluster_version}-x86_64-*"]
+  }
+}

terraform/layer1-aws/aws-eks.tf

@@ -18,15 +18,15 @@ locals {
     {
       "key"                 = "k8s.io/cluster-autoscaler/${local.name}"
       "propagate_at_launch" = "false"
-      "value"               = "true"
+      "value"               = "owned"
     }
   ]
 }
 
 #tfsec:ignore:aws-vpc-no-public-egress-sgr tfsec:ignore:aws-eks-enable-control-plane-logging tfsec:ignore:aws-eks-encrypt-secrets tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "17.1.0"
+  version = "17.3.0"
 
   cluster_name    = local.name
   cluster_version = var.eks_cluster_version
@@ -50,53 +50,115 @@ module "eks" {
     }
   ] : []
 
-  worker_groups_launch_template = [
-    {
-      name                    = "spot"
-      override_instance_types = var.eks_worker_groups.spot.override_instance_types
-      spot_instance_pools     = var.eks_worker_groups.spot.spot_instance_pools
-      asg_max_size            = var.eks_worker_groups.spot.asg_max_size
-      asg_min_size            = var.eks_worker_groups.spot.asg_min_size
-      asg_desired_capacity    = var.eks_worker_groups.spot.asg_desired_capacity
-      subnets                 = module.vpc.private_subnets
-      kubelet_extra_args      = "--node-labels=node.kubernetes.io/lifecycle=spot"
-      public_ip               = false
-      additional_userdata     = file("${path.module}/templates/eks-x86-nodes-userdata.sh")
+  map_roles        = local.eks_map_roles
+  write_kubeconfig = var.eks_write_kubeconfig
+
+  # Create security group rules to allow communication between pods on workers and pods in managed node groups.
+  # Set this to true if you have AWS-Managed node groups and Self-Managed worker groups.
+  # See https://github.com/terraform-aws-modules/terraform-aws-eks/issues/1089
+  worker_create_cluster_primary_security_group_rules = true
+
+  workers_additional_policies = var.eks_workers_additional_policies
 
-      tags = local.worker_tags
+  node_groups_defaults = {
+    ami_type  = "AL2_x86_64"
+    disk_size = 100
+  }
+
+  node_groups = {
+    spot = {
+      desired_capacity = var.node_group_spot.desired_capacity
+      max_capacity     = var.node_group_spot.max_capacity
+      min_capacity     = var.node_group_spot.min_capacity
+      instance_types   = var.node_group_spot.instance_types
+      capacity_type    = var.node_group_spot.capacity_type
+      subnets          = module.vpc.private_subnets
+
+      force_update_version = var.node_group_spot.force_update_version
+
+      k8s_labels = {
+        Environment = local.env
+        nodegroup   = "spot"
+      }
+      additional_tags = {
+        Name = "${local.name}-spot"
+      }
     },
-    {
-      name                 = "ondemand"
-      instance_type        = var.eks_worker_groups.ondemand.instance_type
-      asg_desired_capacity = var.eks_worker_groups.ondemand.asg_desired_capacity
-      subnets              = module.vpc.private_subnets
-      asg_max_size         = var.eks_worker_groups.ondemand.asg_max_size
-      cpu_credits          = "unlimited"
-      kubelet_extra_args   = "--node-labels=node.kubernetes.io/lifecycle=ondemand"
-      public_ip            = false
-      additional_userdata  = file("${path.module}/templates/eks-x86-nodes-userdata.sh")
-
-      tags = local.worker_tags
+    ondemand = {
+      desired_capacity = var.node_group_ondemand.desired_capacity
+      max_capacity     = var.node_group_ondemand.max_capacity
+      min_capacity     = var.node_group_ondemand.min_capacity
+      instance_types   = var.node_group_ondemand.instance_types
+      capacity_type    = var.node_group_ondemand.capacity_type
+      subnets          = module.vpc.private_subnets
+
+      force_update_version = var.node_group_ondemand.force_update_version
+
+      k8s_labels = {
+        Environment = local.env
+        nodegroup   = "ondemand"
+      }
+      additional_tags = {
+        Name = "${local.name}-ondemand"
+      }
     },
+    ci = {
+      desired_capacity = var.node_group_ci.desired_capacity
+      max_capacity     = var.node_group_ci.max_capacity
+      min_capacity     = var.node_group_ci.min_capacity
+      instance_types   = var.node_group_ci.instance_types
+      capacity_type    = var.node_group_ci.capacity_type
+      subnets          = module.vpc.private_subnets
+
+      force_update_version = var.node_group_ci.force_update_version
+
+      k8s_labels = {
+        Environment = local.env
+        nodegroup   = "ci"
+      }
+      additional_tags = {
+        Name = "${local.name}-ci"
+      }
+      taints = [
+        {
+          key    = "nodegroup"
+          value  = "ci"
+          effect = "NO_SCHEDULE"
+      }]
+    }
+  }
+
+  worker_groups_launch_template = [
     {
-      name                    = "ci"
-      override_instance_types = var.eks_worker_groups.ci.override_instance_types
-      spot_instance_pools     = var.eks_worker_groups.ci.spot_instance_pools
-      asg_max_size            = var.eks_worker_groups.ci.asg_max_size
-      asg_min_size            = var.eks_worker_groups.ci.asg_min_size
-      asg_desired_capacity    = var.eks_worker_groups.ci.asg_desired_capacity
-      subnets                 = module.vpc.public_subnets
-      cpu_credits             = "unlimited"
-      kubelet_extra_args      = "--node-labels=node.kubernetes.io/lifecycle=spot --node-labels=purpose=ci --register-with-taints=purpose=ci:NoSchedule"
-      public_ip               = true
-      additional_userdata     = file("${path.module}/templates/eks-x86-nodes-userdata.sh")
+      name                    = "bottlerocket-spot"
+      ami_id                  = data.aws_ami.bottlerocket_ami.id
+      override_instance_types = var.worker_group_bottlerocket.instance_types
+      spot_instance_pools     = var.worker_group_bottlerocket.spot_instance_pools
+      asg_max_size            = var.worker_group_bottlerocket.max_capacity
+      asg_min_size            = var.worker_group_bottlerocket.min_capacity
+      asg_desired_capacity    = var.worker_group_bottlerocket.desired_capacity
+      subnets                 = module.vpc.private_subnets
+      public_ip               = false
+      userdata_template_file  = "${path.module}/templates/userdata-bottlerocket.tpl"
+      userdata_template_extra_args = {
+        enable_admin_container   = false
+        enable_control_container = true
+      }
+      additional_userdata = <<EOT
+[settings.kubernetes.node-labels]
+"eks.amazonaws.com/capacityType" = "SPOT"
+"nodegroup" = "bottlerocket"
+
+[settings.kubernetes.node-taints]
+"nodegroup" = "bottlerocket:NoSchedule"
+EOT
 
       tags = concat(local.worker_tags, [{
-        "key"                 = "k8s.io/cluster-autoscaler/node-template/label/purpose"
+        "key"                 = "k8s.io/cluster-autoscaler/node-template/label/nodegroup"
         "propagate_at_launch" = "true"
-        "value"               = "ci"
+        "value"               = "bottlerocket"
       }])
-    },
+    }
   ]
 
   fargate_profiles = {
@@ -116,7 +178,4 @@ module "eks" {
       })
     }
   }
-
-  map_roles        = local.eks_map_roles
-  write_kubeconfig = var.eks_write_kubeconfig
 }

terraform/layer1-aws/aws-vpc-endpoints.tf

@@ -5,22 +5,25 @@ data "aws_security_group" "default" {
   vpc_id = module.vpc.vpc_id
 }
 
-module "vpc_endpoints" {
+module "vpc_gateway_endpoints" {
   source  = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
   version = "3.2.0"
 
   vpc_id = module.vpc.vpc_id
 
-  security_group_ids = [
-  data.aws_security_group.default.id]
-
   endpoints = {
     s3 = {
-      service = "s3"
+      service      = "s3"
+      service_type = "Gateway"
+      route_table_ids = flatten([
+        module.vpc.intra_route_table_ids,
+        module.vpc.private_route_table_ids,
+        module.vpc.public_route_table_ids
+      ])
       tags = {
-        Name = "s3-vpc-endpoint"
+        Name = "${local.name}-s3"
       }
-    },
+    }
   }
 
   tags = local.tags

terraform/layer1-aws/demo.tfvars.example

@@ -18,30 +18,15 @@ single_nat_gateway = true
 ##########
 # EKS
 ##########
-eks_cluster_version = "1.19"
-
 eks_cluster_encryption_config_enable = true
 
-eks_worker_groups = {
-  spot = {
-    override_instance_types = ["t3.medium", "t3a.medium"]
-    spot_instance_pools     = 2
-    asg_max_size            = 5
-    asg_min_size            = 0
-    asg_desired_capacity    = 1
-  },
-  ondemand = {
-    instance_type        = "t3a.medium"
-    asg_desired_capacity = 1
-    asg_max_size         = 6
-  },
-  ci = {
-    override_instance_types = ["t3.medium", "t3a.medium"]
-    spot_instance_pools     = 2
-    asg_max_size            = 3
-    asg_min_size            = 0
-    asg_desired_capacity    = 0
-  }
+node_group_ondemand = {
+  instance_types       = ["m5a.medium"]
+  capacity_type        = "ON_DEMAND"
+  max_capacity         = 5
+  min_capacity         = 1
+  desired_capacity     = 1
+  force_update_version = false
 }
 
 eks_write_kubeconfig = false

terraform/layer1-aws/templates/eks-x86-nodes-userdata.sh

@@ -0,0 +1,26 @@
+# https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#description-of-settings
+${pre_userdata}
+[settings.kubernetes]
+api-server = "${endpoint}"
+cluster-certificate = "${cluster_auth_base64}"
+cluster-name = "${cluster_name}"
+
+${additional_userdata}
+
+# Hardening based on https://github.com/bottlerocket-os/bottlerocket/blob/develop/SECURITY_GUIDANCE.md
+
+# Enable kernel lockdown in "integrity" mode.
+# This prevents modifications to the running kernel, even by privileged users.
+[settings.kernel]
+lockdown = "integrity"
+
+# The admin host container provides SSH access and runs with "superpowers".
+# It is disabled by default, but can be disabled explicitly.
+[settings.host-containers.admin]
+enabled = ${enable_admin_container}
+
+# The control host container provides out-of-band access via SSM.
+# It is enabled by default, and can be disabled if you do not expect to use SSM.
+# This could leave you with no way to access the API and change settings on an existing node!
+[settings.host-containers.control]
+enabled = ${enable_control_container}