E2E Testing about digitalocean-cloud-controller-manager HOT 16 CLOSED

diff --git a/01-master.yaml b/01-master.yaml
index 97cf2f5..2bbce16 100644
--- a/01-master.yaml
+++ b/01-master.yaml
@@ -36,12 +36,14 @@ write_files:
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ssl/ca.pem \
-          --api-servers=http://127.0.0.1:8080 \
+          --kubeconfig=/etc/kubernetes/kubelet-kubeconfig.yaml \
+          --require-kubeconfig \
           --network-plugin-dir=/etc/kubernetes/cni/net.d \
+          --register-schedulable=false \
+          --cloud-provider=external \
           --container-runtime=docker \
           --allow-privileged=true \
           --pod-manifest-path=/etc/kubernetes/manifests \
-          --hostname-override=$private_ipv4 \
           --cluster-dns=${DNS_SERVICE_IP} \
           --cluster-domain=cluster.local \
           --node-labels=kubernetes.io/role=master \
@@ -52,6 +54,23 @@ write_files:
 
         [Install]
         WantedBy=multi-user.target
+  - path: "/etc/kubernetes/kubelet-kubeconfig.yaml"
+    permissions: "0755"
+    content: |
+      apiVersion: v1
+      kind: Config
+      clusters:
+      - name: local
+        cluster:
+          server: http://127.0.0.1:8080
+      users:
+      - name: kubelet
+      contexts:
+      - context:
+          cluster: local
+          user: kubelet
+        name: kubelet-context
+      current-context: kubelet-context
   - path: "/etc/kubernetes/manifests/kube-apiserver.yaml"
     permissions: "0755"
     content: |
@@ -79,6 +98,8 @@ write_files:
             - --service-cluster-ip-range=${SERVICE_IP_RANGE}
             - --secure-port=443
             - --storage-backend=etcd2
+            - --cloud-provider=external
+            - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
             - --advertise-address=$private_ipv4
             - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
             - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
@@ -161,6 +182,7 @@ write_files:
             - --master=http://127.0.0.1:8080
             - --leader-elect=true
             - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
+            - --cloud-provider=external
             - --root-ca-file=/etc/kubernetes/ssl/ca.pem
             livenessProbe:
               httpGet:
diff --git a/02-worker.yaml b/02-worker.yaml
index 44ae8af..0a4be27 100644
--- a/02-worker.yaml
+++ b/02-worker.yaml
@@ -37,13 +37,13 @@ write_files:
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ssl/ca.pem \
-          --api-servers=https://${MASTER_HOST} \
+          --require-kubeconfig \
           --network-plugin-dir=/etc/kubernetes/cni/net.d \
           --container-runtime=docker \
           --register-node=true \
           --allow-privileged=true \
+          --cloud-provider=external \
           --pod-manifest-path=/etc/kubernetes/manifests \
-          --hostname-override=$private_ipv4 \
           --cluster-dns=${DNS_SERVICE_IP} \
           --cluster-domain=cluster.local \
           --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \
@@ -104,6 +104,7 @@ write_files:
         clusters:
         - name: local
           cluster:
+            server: https://${MASTER_HOST}
             certificate-authority: /etc/kubernetes/ssl/ca.pem
         users:
         - name: kubelet
diff --git a/05-do-secret.yaml b/05-do-secret.yaml
new file mode 100644
index 0000000..9d89afe
--- /dev/null
+++ b/05-do-secret.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: digitalocean
+  namespace: kube-system
+data:
+  # insert your base64 encoded DO access token here, ensure there's no trailing newline:
+  # to base64 encode your token run:
+  #      echo -n "abc123abc123doaccesstoken" | base64
+  access-token: "$DO_ACCESS_TOKEN_BASE64"
diff --git a/deploy.tf b/deploy.tf
index 9b31387..01f195e 100644
--- a/deploy.tf
+++ b/deploy.tf
@@ -23,7 +23,7 @@ variable "ssh_private_key" {
 
 variable "number_of_workers" {}
 variable "hyperkube_version" {
-    default = "v1.7.3_coreos.0"
+    default = "v1.8.0_coreos.0"
 }
 
 variable "prefix" {
@@ -435,3 +435,16 @@ resource "null_resource" "deploy_microbot" {
 EOF
     }
 }
+
+resource "null_resource" "deploy_digitalocean_cloud_controller_manager" {
+    depends_on = ["null_resource.setup_kubectl"]
+    provisioner "local-exec" {
+        command = <<EOF
+            TOKEN=$(echo "${var.do_token}" | tr -d '\n' | base64)
+            sed -e "s/\$DO_ACCESS_TOKEN_BASE64/$TOKEN/" < ${path.module}/05-do-secret.yaml > ./secrets/05-do-secret.rendered.yaml
+            until kubectl get pods 2>/dev/null; do printf '.'; sleep 5; done
+            kubectl create -f ./secrets/05-do-secret.rendered.yaml
+            kubectl create -f https://raw.githubusercontent.com/digitalocean/digitalocean-cloud-controller-manager/master/releases/v0.1.2.yml
+EOF
+    }
+}

ssh-keygen # to tmp file
<pull token from env DO_TOKEN, and configure doctl>
ID="$(head -10 /dev/urandom | sha512sum | cut -b 1-30)"
trap "doctl remove all droplet with tag ID=$ID" EXIT

doctl compute droplet create master --image coreos-stable --size 2gb --wait --tag "$ID" --ssh-keys <tmp ssh file>
./init-master.sh <ip>

# Add DO CCM here or in init-master..

doctl compute droplet create worker1--image coreos-stable --size 2gb --wait --tag "$ID" --ssh-keys <tmp ssh file>
./init-node.sh <ip> <kube-config>

doctl compute droplet create worker2 --image coreos-stable --size 2gb --wait --tag "$ID" --ssh-keys <tmp ssh file>
./init-node.sh <ip> <kube-config>
until kubectl get node == 3 ready nodes; # TODO: Add timeout
  sleep 5
done

# Run e2e test

for node in nodes; do
    #check
    failure-domain.beta.kubernetes.io/region == doctl get node region
    beta.kubernetes.io/instance-type == doctl get type
    InternalIP == doctl get private ipv4
    ExternalIP == doctl get public ipv4
done
doctl delete node worker2
timeout 120 until kubectl get node worker2 gone; do
    sleep 5
done

# Maybe ignore https example for now?
for manifest in examples/loadbalancers/*.yml; do
    kubectl create -f $manifest
    timeout 120 until kubectl get loadbalancer ready; do
        sleep 5
    done
    timeout 120 until curl success; do
        sleep 5
    done
    kubectl delete -f $manifest
    timeout 120 until kubectl get loadbalancer gone; do
        sleep 5
    done
    doctl check loadbalancer is gone
done

...
# Execute all playbooks
resource "null_resource" "execute_playbooks" {
        depends_on      = ["null_resource.write_digitalocean_token"]
        triggers {
                rendered_inventory = "${template_file.ansible_inventory.rendered}"
        }
        provisioner "local-exec" {
                command = <<EOF
                    export ANSIBLE_HOST_KEY_CHECKING=False
                    ...
                    ansible-playbook -v -i ${var.output_path}/inventory ${path.module}/playbooks/myplaybook.yml
                    ...
EOF
        }
}
...

---
- hosts: master
  become: yes
  tasks:

##### Secrets

    - name: 'Copy digitalocean secret yml to master node'
      copy:
        src: "{{ ath_output_path }}/do_secret.rendered.yml"
        dest: /home/valkyrie/do_secret.yml
        owner: valkyrie
        mode: 0755

    - name: 'Install DO secret'
      become: yes
      become_user: valkyrie
      shell: kubectl create -f $HOME/do_secret.yml >> /home/valkyrie/do_secrets_installed.txt
      args:
        creates: /home/valkyrie/do_secrets_installed.txt

##### Cloud Controller Manager (CCM)

    - name: 'Copy CCM yml to master node'
      copy:
        src: ./assets/do_ccm.yml
        dest: /home/valkyrie/do_ccm.yml
        owner: valkyrie
        mode: 0755

    - name: 'Check if CCM was installed'
      stat:
        path: /home/valkyrie/ccm_installed.txt
      register: ccm_installed

    - name: 'Patch CCM'
      become: yes
      become_user: valkyrie
      shell: kubectl replace --force -f $HOME/do_ccm.yml
      when: ccm_installed.stat.exists == True

    - name: 'Install CCM'
      become: yes
      become_user: valkyrie
      shell: kubectl create -f $HOME/do_ccm.yml >> /home/valkyrie/ccm_installed.txt
      when: ccm_installed.stat.exists == False

##### Cloud Storage Interface (CSI)

    - name: 'Copy CSI yml to master node'
      copy:
        src: ./assets/do_csi.yml
        dest: /home/valkyrie/do_csi.yml
        owner: valkyrie
        mode: 0755

    - name: 'Check if CSI was installed'
      stat:
        path: /home/valkyrie/csi_installed.txt
      register: csi_installed

    - name: 'Install CSI'
      become: yes
      become_user: valkyrie
      shell: kubectl create -f $HOME/do_csi.yml >> /home/valkyrie/csi_installed.txt
      when: csi_installed.stat.exists == False