Setup flannel overlay

README.md

@@ -51,3 +51,24 @@ When all Pis running and the proper IPs are entered you can re-run ansible with
 #### Final words
 
 Bear with me, that my first Ansible playbook ;-)
+
+-------
+
+## OS X Nat
+
+* Enable IP forwarding:
+
+      sudo sysctl -w net.inet.ip.forwarding=1
+      sudo sysctl -w net.inet.ip.fw.enable=1
+
+* Create a pfctl rule in a file `nat-rules` with (en0: Interface with your connected IP, en5: WiFi connected to WiFi route):
+
+      nat on en0 from en5 to any -> (en0)
+
+* Apply the rule:
+
+      sudo pfctl -d #disables pfctl
+      sudo pfctl -F all #flushes all pfctl rules
+      sudo pfctl -f ./nat-rules -e #starts pfctl and loads the rules from the nat-rules file
+
+* Ensure in your WiFi Router that the nodes get the OS-X's IP 192.168.23.100 as router (either via DHCP or in the static routing)

kubernetes-playbook.yml

@@ -8,7 +8,8 @@
   roles:
   - role: kubernetes
     mode: master
-    master: "master"
+    # "master" must be an IP, otherwise flannel will core dump:
+    master: 192.168.23.200
 
 - name: Kubernetes Nodes
 
@@ -19,4 +20,4 @@
   roles:
   - role: kubernetes
     mode: node
-    master: "master"
+    master: 192.168.23.200

restart-docker-playbook.yml

@@ -0,0 +1,10 @@
+- name: Start Docker daemons
+
+  hosts: pis
+  remote_user: pi
+  become: true
+  become_method: sudo
+
+  tasks:
+  - name: Enable Docker
+    service: name=docker enabled=yes state=started

roles/docker/tasks/main.yml

@@ -25,12 +25,15 @@
 - name: Add user pi to group docker
   user: name=pi groups=docker append=yes
 
-- name: Enable remote port for Docker
+- name: Restart on Failure
   lineinfile:
-    dest: /etc/default/docker
-    regexp: '^\s*DOCKER_OPTS'
-    line: 'DOCKER_OPTS="-H tcp://{{ inventory_hostname }}:2375 -H unix:///var/run/docker.sock --storage-driver=overlay -D"'
+    dest: /lib/systemd/system/docker.service
+    insertafter: '\[Service\]'
+    line: "{{ item }}"
     state: present
+  with_items:
+    - "Restart=on-failure"
+    - "RestartSec=5"
 
 - name: Enable Docker
   service: name=docker enabled=yes state=started

roles/init/tasks/main.yml

@@ -47,3 +47,4 @@
   with_items:
     - hdparm
     - iperf
+    - mtr-tiny

roles/init/templates/hosts

@@ -5,9 +5,8 @@
 ff02::1         ip6-allnodes
 ff02::2         ip6-allrouters
 
-127.0.0.1       {{ name }} {{ host_extra if host_extra is defined else ''}}
 {% for item in groups['pis'] %}
-{% if hostvars[item].ansible_default_ipv4.address and name != hostvars[item].name %}
+{% if hostvars[item].ansible_default_ipv4.address %}
 {{ hostvars[item].ansible_default_ipv4.address }} {{ hostvars[item].name }}{% if hostvars[item].host_extra is defined %} {{ hostvars[item].host_extra }}{% endif %}
 
 {% endif %}

roles/kubernetes/files/binaries/README.md

@@ -0,0 +1,9 @@
+# Binaries used for Kubernetes
+
+The following binaries are started directly in addition to docker
+to spane a Kubernetes cluster:
+
+* kubectl 1.2.0
+* kubelet 1.2.0
+* etcd 2.2.5
+* flanneld 0.5.5

roles/kubernetes/tasks/etcd.yml

@@ -0,0 +1,14 @@
+- name: Install etcd
+  copy: src=binaries/etcd-{{ version.etcd }}/{{ item}} dest=/usr/bin/{{ item }} owner=root group=root mode=0755
+  with_items:
+    - etcd
+    - etcdctl
+
+- name: Create etcd data dir directory
+  file: path=/var/lib/etcd state=directory recurse=false
+
+- name: Install etcd service definition
+  template: src=master/etcd.service dest=/lib/systemd/system/etcd.service owner=root group=root mode=0644
+
+- name: Enable and start etcd service
+  service: name=etcd enabled=true state=started

roles/kubernetes/tasks/flanneld.yml

@@ -0,0 +1,14 @@
+- name: Install flanneld
+  copy: src=binaries/flannel-{{ version.flannel }}/flanneld dest=/usr/bin/flanneld owner=root group=root mode=0755
+
+- name: Create flannel env directory
+  file: path=/var/lib/flannel state=directory recurse=false
+
+- name: Add flannel_init.sh for setting flannel network
+  template: src={{ mode }}/flannel_init.sh dest=/etc/kubernetes/flannel_init.sh mode=0755
+
+- name: Install flannel service definition
+  template: src={{ mode }}/flannel.service dest=/lib/systemd/system/flannel.service owner=root group=root mode=0644
+
+- name: Enable and start flannel service
+  service: name=flannel enabled=true state=started

roles/kubernetes/tasks/main.yml

@@ -1,24 +1,36 @@
 - name: Install kubelet and kubectl
-  copy: src=binaries/{{ version.kubernetes }}/{{ item}} dest=/usr/bin/{{ item }} owner=root group=root mode=0755
+  copy: src=binaries/kubernetes-{{ version.kubernetes }}/{{ item}} dest=/usr/bin/{{ item }} owner=root group=root mode=0755
   with_items:
     - kubelet
     - kubectl
 
 - name: Create Kubernetes config directory
   file: path=/etc/kubernetes/manifests/ state=directory recurse=no owner=root group=root mode=0755
 
-- name: Create etcd data directory
-  file: path=/var/lib/etcd state=directory recurse=false
+- name: Add etcd
+  include: etcd.yml
   when: mode == "master"
 
+- name: Add flanneld
+  include: flanneld.yml
+
+- name: Add bridge-utils
+  apt: name='bridge-utils' state=present force=yes
+
+- name: Install own docker service
+  template: src=docker.service dest=/lib/systemd/system/docker.service owner=root group=root mode=0644
+
+- name: Enable and restart docker service
+  service: name=docker enabled=true state=started
+
 - name: Copy configuration
-  template: src=kubernetes-{{ mode }}.yml dest=/etc/kubernetes/manifests/kubernetes.yaml
+  template: src={{ mode }}/kubernetes.yml dest=/etc/kubernetes/manifests/kubernetes.yaml
 
 - name: Setup profile
   template: src=kubernetes_profile.sh dest=/etc/profile.d/kubernetes.sh mode=0644
-  
+
 - name: Install kubelet service definition
-  template: src=kubelet.service dest=/etc/systemd/system/kubelet.service owner=root group=root mode=0644
+  template: src=kubelet.service dest=/lib/systemd/system/kubelet.service owner=root group=root mode=0644
 
 - name: Enable and start kubelet service
-  service: name=kubelet enabled=true state=restarted
+  service: name=kubelet enabled=true state=started

roles/kubernetes/templates/docker.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network.target docker.socket flannel.service
+Requires=docker.socket flannel.service
+
+[Service]
+Type=notify
+# Flannel bridge to use by Docker:
+EnvironmentFile=/var/lib/flannel/subnet.env
+ExecStartPre=-/sbin/ifconfig docker0 down
+ExecStartPre=-/sbin/brctl delbr docker0
+ExecStart=/usr/bin/docker daemon \
+    --bip=${FLANNEL_SUBNET} \
+    --mtu=${FLANNEL_MTU} \
+    -H tcp://{{ inventory_hostname }}:2375 \
+    -H unix:///var/run/docker.sock \
+    --storage-driver=overlay \
+    --exec-opt native.cgroupdriver=cgroupfs
+Restart=on-failure
+RestartSec=5
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/kubelet.service

@@ -1,14 +1,16 @@
 [Unit]
 Description=Kubernetes Kubelet
 Documentation=https://github.com/kubernetes/kubernetes
+After=network.target docker.service
+Requires=docker.service
 
 [Service]
 ExecStart=/usr/bin/kubelet  \
---api-servers=http://{{ master }}:8080 \
---allow-privileged=true \
---pod_infra_container_image={{ images.pause }} \
---config=/etc/kubernetes/manifests \
---v=2
+  --api-servers=http://{{ master }}:8080 \
+  --allow-privileged=true \
+  --pod_infra_container_image={{ images.pause }} \
+  --config=/etc/kubernetes/manifests \
+  --v={{ debug_level }}
 Restart=on-failure
 RestartSec=5
 

roles/kubernetes/templates/master/etcd.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=etcd Config Store
+Documentation=https://github.com/coreos/etcd
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/etcd  \
+  --data-dir=/var/lib/etcd \
+  --advertise-client-urls=http://{{ master }}:{{ etcd.port }} \
+  --listen-client-urls=http://{{ master }}:{{ etcd.port }} \
+  --listen-peer-urls=http://{{ master }}:{{ etcd.peer_port }} \
+  --name=etcd
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/master/flannel.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Flannel Overlay Network for Kubernetes
+After=network.target etcd.service
+Requires=etcd.service
+
+[Service]
+ExecStartPre=/etc/kubernetes/flannel_init.sh
+ExecStart=/usr/bin/flanneld \
+     --etcd-endpoints=http://{{ master }}:{{ etcd.port }} \
+     --subnet-file=/var/lib/flannel/subnet.env
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/master/flannel_init.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+etcd_url="http://{{ master }}:{{ etcd.port}}"
+pod_subnet="{{ network.pod_subnet }}"
+while [ $(curl -fs "${etcd_url}/v2/machines" 2>&1 1>/dev/null; echo $?) != 0 ]
+do
+  sleep 1
+done
+
+network_config=$(cat <<EOT
+{
+  "Network": "${pod_subnet}",
+  "Backend": {
+     "Type": "host-gw"
+  }
+}
+EOT
+)
+echo "Setting network config for flannel: "
+/usr/bin/etcdctl \
+   --endpoint ${etcd_url} \
+  set /coreos.com/network/config "$network_config"

roles/kubernetes/templates/kubernetes-master.yml → roles/kubernetes/templates/master/kubernetes.yml

@@ -4,55 +4,38 @@ metadata:
   name: kube-master
 spec:
   hostNetwork: true
-  volumes:
-  - name: "etcd-datadir"
-    hostPath:
-      path: /var/lib/etcd
   containers:
-    - name: etcd
-      image: "{{ images.etcd }}"
-      args:
-        - "/usr/local/bin/etcd"
-        - "--data-dir=/var/etcd/data"
-        - "--advertise-client-urls=http://127.0.0.1:2379"
-        - "--listen-client-urls=http://127.0.0.1:2379"
-        - "--listen-peer-urls=http://127.0.0.1:2380"
-        - "--name=etcd"
-      volumeMounts:
-        - name: "etcd-datadir"
-          mountPath: /var/lib/etcd
-          readOnly: false
     - name: "kube-apiserver"
       image: "{{ images.hyperkube }}"
       args:
         - "/hyperkube"
         - "apiserver"
         - "--allow-privileged=true"
-        - "--etcd-servers=http://127.0.0.1:2379"
+        - "--etcd-servers=http://{{ master }}:{{ etcd.port }}"
         - "--insecure-bind-address=0.0.0.0"
-        - "--service-cluster-ip-range=10.200.100.0/24"
+        - "--service-cluster-ip-range={{ network.service_subnet }}"
         - "--service-node-port-range=30000-37000"
-        - "--v=2"
+        - "--v={{ debug_level }}"
     - name: "kube-controller-manager"
       image: "{{ images.hyperkube }}"
       args:
         - "/hyperkube"
         - "controller-manager"
         - "--master=http://127.0.0.1:8080"
-        - "--v=2"
+        - "--v={{ debug_level }}"
     - name: "kube-scheduler"
       image: "{{ images.hyperkube }}"
       args:
         - "/hyperkube"
         - "scheduler"
         - "--master=http://127.0.0.1:8080"
-        - "--v=2"
+        - "--v={{ debug_level }}"
     - name: "kube-proxy"
       image: "{{ images.hyperkube }}"
       args:
         - "/hyperkube"
         - "proxy"
         - "--master=http://127.0.0.1:8080"
-        - "--v=2"
+        - "--v={{ debug_level }}"
       securityContext:
         privileged: true

roles/kubernetes/templates/node/flannel.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Flannel Overlay Network for Kubernetes
+After=network.target
+
+[Service]
+ExecStartPre=/etc/kubernetes/flannel_init.sh
+ExecStart=/usr/bin/flanneld \
+     --etcd-endpoints=http://{{ master }}:{{ etcd.port }} \
+     --subnet-file=/var/lib/flannel/subnet.env
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/node/flannel_init.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+etcd_url="http://{{ master }}:{{ etcd.port}}"
+pod_subnet="{{ network.pod_subnet }}"
+while [ $(curl -fs "${etcd_url}/v2/machines" 2>&1 1>/dev/null; echo $?) != 0 ]
+do
+  sleep 1
+done

roles/kubernetes/templates/kubernetes-node.yml → roles/kubernetes/templates/node/kubernetes.yml

@@ -11,6 +11,6 @@ spec:
         - "/hyperkube"
         - "proxy"
         - "--master=http://{{ master }}:8080"
-        - "--v=2"
+        - "--v={{ debug_level }}"
       securityContext:
         privileged: true

roles/kubernetes/vars/main.yml

@@ -1,6 +1,16 @@
 version:
   kubernetes: v1.2.0
+  flannel: 0.5.5
+  etcd: 2.2.5
 images:
   hyperkube: gcr.io/google-containers/hyperkube-arm:{{ version.kubernetes }}
-  etcd: gcr.io/google-containers/etcd-arm:2.2.1
   pause: gcr.io/google-containers/pause-arm:2.0
+flannel:
+  backend: host-gw
+etcd:
+  port: 2379
+  peer_port: 2380
+network:
+  service_subnet: 10.200.100.0/24
+  pod_subnet: 10.1.0.0/16
+debug_level: 2