WIP: Run Cloud Controller Manager on AWS

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -1,6 +1,18 @@
 ---
 systemd:
   units:
+    - name: set-hostname.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Set the hostname programmitcally for the AWS-preferred format
+        After=network.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'hostnamectl set-hostname $(curl -s http://169.254.169.254/latest/meta-data/hostname)'
+        [Install]
+        RequiredBy=kubelet.service
     - name: etcd-member.service
       enable: true
       dropins:
@@ -52,6 +64,7 @@ systemd:
         [Unit]
         Description=Kubelet via Hyperkube
         Wants=rpc-statd.service
+        Wants=set-hostname.service
         [Service]
         ConditionPathExists=/etc/kubernetes/kubeconfig
         EnvironmentFile=/etc/kubernetes/kubelet.env
@@ -85,6 +98,7 @@ systemd:
           --authentication-token-webhook \
           --authorization-mode=Webhook \
           --client-ca-file=/etc/kubernetes/ca.crt \
+          --cloud-provider=external \
           --cluster_dns=${cluster_dns_service_ip} \
           --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/cni/net.d \

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager.tf

@@ -0,0 +1,8 @@
+resource "local_file" "cloud_controller_manager" {
+  filename = "${var.asset_dir}/charts/kube-system/cloud-controller-manager.yaml"
+}
+
+resource "template_dir" "cloud_controller_manager" {
+  source_dir      = "${path.module}/cloud-controller-manager"
+  destination_dir = "${var.asset_dir}/charts/kube-system/cloud-controller-manager"
+}

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: cloud-controller-manager
+version: 0.1.0

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager/templates/cloud-controller-manager-rbac.yaml

@@ -0,0 +1,61 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:controller:cloud-controller-manager
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - persistentvolumes
+  - nodes
+  - nodes/status
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:controller:shared-informers
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: shared-informers
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:controller:pvl-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: pvl-controller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:controller:cloud-node-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-node-controller
+  namespace: kube-system

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager/templates/cloud-controller-manager.yaml

@@ -0,0 +1,75 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: cloud-controller-manager
+  name: cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-controller-manager
+    spec:
+      serviceAccountName: cloud-controller-manager
+      containers:
+      - name: cloud-controller-manager
+        image: k8s.gcr.io/cloud-controller-manager:v1.8.0
+        command:
+        - /usr/local/bin/cloud-controller-manager
+        - --cloud-provider=aws
+        - --leader-elect=true
+        - --use-service-account-credentials
+        # Prevents breakdown of CCM spamming AWS
+        # for cloud routes
+        - --configure-cloud-routes=false
+        - --allocate-node-cidrs=true
+        - --cluster-cidr=172.17.0.0/16
+        - --allow-untagged-cloud
+        volumeMounts:
+        - mountPath: /etc/ssl/certs
+          name: ca-certs
+          readOnly: true
+        - mountPath: /usr/chare/ca-certificates/
+          name: ca-real-certs
+          readOnly: true
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      volumes:
+        - hostPath:
+            path: /usr/share/ca-certificates/
+            type: DirectoryOrCreate
+          name: ca-real-certs
+        - hostPath:
+            path: /usr/share/ca-certificates/
+            type: DirectoryOrCreate
+          name: ca-certs
+      nodeSelector:
+        node.kubernetes.io/master: ""

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cloud-controller-manager/values.yaml

@@ -0,0 +1,68 @@
+# Default values for cloud-controller-manager.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: nginx
+  tag: stable
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 80
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/ssh.tf

@@ -79,6 +79,7 @@ resource "null_resource" "bootkube-start" {
     module.bootkube,
     aws_route53_record.apiserver,
     null_resource.copy-controller-secrets,
+    local_file.cloud_controller_manager,
   ]
 
   connection {

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -1,6 +1,18 @@
 ---
 systemd:
   units:
+    - name: set-hostname.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Set the hostname programmitcally for the AWS-preferred format
+        After=network.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'hostnamectl set-hostname $(curl -s http://169.254.169.254/latest/meta-data/hostname)'
+        [Install]
+        RequiredBy=kubelet.service
     - name: docker.service
       enable: true
     - name: iscsid.service
@@ -27,6 +39,7 @@ systemd:
         [Unit]
         Description=Kubelet via Hyperkube
         Wants=rpc-statd.service
+        Wants=set-hostname.service
         [Service]
         EnvironmentFile=/etc/kubernetes/kubelet.env
         Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \

assets/lokomotive-kubernetes/bootkube/resources/charts/kube-apiserver/templates/kube-apiserver.yaml

@@ -86,7 +86,9 @@ spec:
           --authorization-mode=RBAC \
           --bind-address=$(cat /run/kube-apiserver/address) \
           --client-ca-file=/etc/kubernetes/secrets/ca.crt \
+          {{- if .Values.apiserver.cloudProvider }}
           --cloud-provider={{ .Values.apiserver.cloudProvider }} \
+          {{- end }}
           --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy \
           --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt \
           --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt \
@@ -122,7 +124,9 @@ spec:
         - --authorization-mode=RBAC
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+        {{- if .Values.apiserver.cloudProvider }}
         - --cloud-provider={{ .Values.apiserver.cloudProvider }}
+        {{- end }}
         - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy
         - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
         - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt

assets/lokomotive-kubernetes/bootkube/resources/charts/kubelet/templates/kubelet-ds.yaml

@@ -35,6 +35,9 @@ spec:
           --authentication-token-webhook \
           --authorization-mode=Webhook \
           --client-ca-file=/etc/kubernetes/ca.crt \
+          {{- if .Values.cloudProvider  }}
+          --cloud-provider={{ .Values.cloudProvider }} \
+          {{- end }}
           --cluster_dns={{ .Values.clusterDNS }} \
           --cluster_domain={{ .Values.clusterDomain }} \
           --cni-conf-dir=/etc/cni/net.d \

assets/lokomotive-kubernetes/bootkube/resources/charts/kubelet/values.yaml

@@ -1,3 +1,4 @@
 image: k8s.gcr.io/hyperkube:v1.18.3
 clusterDNS: 10.0.0.10
 clusterDomain: cluster.local
+cloudProvider: "external"