Private Network Interfaces (kubernetes-digitalocean-terraform#18)

* Access etcd via private network

* Use private network for master-cluster

Author: alpe

00-etcd.yaml

@@ -2,7 +2,7 @@
 
 coreos:
   etcd2:
-    advertise-client-urls: https://$public_ipv4:2379
+    advertise-client-urls: https://$private_ipv4:2379 # multi-region and multi-cloud deployments need to use $public_ipv4
     listen-client-urls: https://0.0.0.0:2379
     client-cert-auth: true
     trusted-ca-file: /etc/kubernetes/ssl/ca.pem

01-master.yaml

@@ -4,7 +4,7 @@ write_files:
   - path: "/etc/flannel/options.env"
     permissions: "0755"
     content: |
-        FLANNELD_IFACE=$public_ipv4
+        FLANNELD_IFACE=$private_ipv4
         FLANNELD_ETCD_ENDPOINTS=https://${ETCD_IP}:2379
         FLANNELD_ETCD_CAFILE=/etc/ssl/etcd/ca.pem
         FLANNELD_ETCD_CERTFILE=/etc/ssl/etcd/client.pem
@@ -39,7 +39,7 @@ write_files:
           --register-schedulable=false \
           --allow-privileged=true \
           --config=/etc/kubernetes/manifests \
-          --hostname-override=$public_ipv4 \
+          --hostname-override=$private_ipv4 \
           --cluster-dns=${DNS_SERVICE_IP} \
           --cluster-domain=cluster.local
         Restart=always
@@ -70,7 +70,7 @@ write_files:
             - --allow-privileged=true
             - --service-cluster-ip-range=${SERVICE_IP_RANGE}
             - --secure-port=443
-            - --advertise-address=$public_ipv4
+            - --advertise-address=$private_ipv4
             - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
             - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
             - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem

02-worker.yaml

@@ -5,7 +5,7 @@ write_files:
   - path: "/etc/flannel/options.env"
     permissions: "0755"
     content: |
-        FLANNELD_IFACE=$public_ipv4
+        FLANNELD_IFACE=$private_ipv4
         FLANNELD_ETCD_ENDPOINTS=https://${ETCD_IP}:2379
         FLANNELD_ETCD_CAFILE=/etc/ssl/etcd/ca.pem
         FLANNELD_ETCD_CERTFILE=/etc/ssl/etcd/worker.pem
@@ -40,7 +40,7 @@ write_files:
           --register-node=true \
           --allow-privileged=true \
           --config=/etc/kubernetes/manifests \
-          --hostname-override=$public_ipv4 \
+          --hostname-override=$private_ipv4 \
           --cluster-dns=${DNS_SERVICE_IP} \
           --cluster-domain=cluster.local \
           --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \

deploy.tf

@@ -44,6 +44,7 @@ resource "digitalocean_droplet" "k8s_etcd" {
     image = "coreos-stable"
     name = "k8s-etcd"
     region = "${var.do_region}"
+    private_networking = true
     size = "512mb"
     user_data = "${file("00-etcd.yaml")}"
     ssh_keys = [
@@ -60,7 +61,7 @@ EOF
     # Generate k8s-etcd server certificate
     provisioner "local-exec" {
         command = <<EOF
-            $PWD/cfssl/generate_server.sh k8s_etcd ${digitalocean_droplet.k8s_etcd.ipv4_address}
+            $PWD/cfssl/generate_server.sh k8s_etcd ${digitalocean_droplet.k8s_etcd.ipv4_address_private}
 EOF
     }
 
@@ -122,7 +123,7 @@ data "template_file" "master_yaml" {
     template = "${file("01-master.yaml")}"
     vars {
         DNS_SERVICE_IP = "10.3.0.10"
-        ETCD_IP = "${digitalocean_droplet.k8s_etcd.ipv4_address}"
+        ETCD_IP = "${digitalocean_droplet.k8s_etcd.ipv4_address_private}"
         POD_NETWORK = "10.2.0.0/16"
         SERVICE_IP_RANGE = "10.3.0.0/24"
         HYPERCUBE_VERSION = "${var.hypercube_version}"
@@ -141,6 +142,7 @@ resource "digitalocean_droplet" "k8s_master" {
     image = "coreos-stable"
     name = "k8s-master"
     region = "${var.do_region}"
+    private_networking = true
     size = "512mb"
     user_data = "${data.template_file.master_yaml.rendered}"
     ssh_keys = [
@@ -150,7 +152,7 @@ resource "digitalocean_droplet" "k8s_master" {
     # Generate k8s_master server certificate
     provisioner "local-exec" {
         command = <<EOF
-            $PWD/cfssl/generate_server.sh k8s_master "${digitalocean_droplet.k8s_master.ipv4_address},10.3.0.1,kubernetes.default,kubernetes"
+            $PWD/cfssl/generate_server.sh k8s_master "${digitalocean_droplet.k8s_master.ipv4_address},${digitalocean_droplet.k8s_master.ipv4_address_private},10.3.0.1,kubernetes.default,kubernetes"
 EOF
     }
 
@@ -177,7 +179,7 @@ EOF
         }
     }
 
-    # Generate k9s_master client certificate
+    # Generate k8s_master client certificate
     provisioner "local-exec" {
         command = <<EOF
             $PWD/cfssl/generate_client.sh k8s_master
@@ -218,7 +220,7 @@ EOF
     provisioner "remote-exec" {
         inline = [
             "sudo systemctl daemon-reload",
-            "curl --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/client.pem --key /etc/kubernetes/ssl/client-key.pem -X PUT -d 'value={\"Network\":\"10.2.0.0/16\",\"Backend\":{\"Type\":\"vxlan\"}}' https://${digitalocean_droplet.k8s_etcd.ipv4_address}:2379/v2/keys/coreos.com/network/config",
+            "curl --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/client.pem --key /etc/kubernetes/ssl/client-key.pem -X PUT -d 'value={\"Network\":\"10.2.0.0/16\",\"Backend\":{\"Type\":\"vxlan\"}}' https://${digitalocean_droplet.k8s_etcd.ipv4_address_private}:2379/v2/keys/coreos.com/network/config",
             "sudo systemctl start flanneld",
             "sudo systemctl enable flanneld",
             "sudo systemctl start kubelet",
@@ -244,8 +246,8 @@ data "template_file" "worker_yaml" {
     template = "${file("02-worker.yaml")}"
     vars {
         DNS_SERVICE_IP = "10.3.0.10"
-        ETCD_IP = "${digitalocean_droplet.k8s_etcd.ipv4_address}"
-        MASTER_HOST = "${digitalocean_droplet.k8s_master.ipv4_address}"
+        ETCD_IP = "${digitalocean_droplet.k8s_etcd.ipv4_address_private}"
+        MASTER_HOST = "${digitalocean_droplet.k8s_master.ipv4_address_private}"
         HYPERCUBE_VERSION = "${var.hypercube_version}"
     }
 }
@@ -260,11 +262,11 @@ data "template_file" "worker_yaml" {
 
 resource "digitalocean_droplet" "k8s_worker" {
     count = "${var.number_of_workers}"
-
     image = "coreos-stable"
     name = "${format("k8s-worker-%02d", count.index + 1)}"
     region = "${var.do_region}"
     size = "512mb"
+    private_networking = true
     user_data = "${data.template_file.worker_yaml.rendered}"
     ssh_keys = [
         "${var.ssh_fingerprint}"