get-addon-templates

#!/usr/bin/env python3

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import logging
import yaml
from contextlib import contextmanager


description = """
Get addon templates for the snap.

This will clone the kubernetes repo and place the addons in ./templates
"""

logging.basicConfig(stream=sys.stdout, level=logging.INFO)
log = logging.getLogger(__name__)


def run_with_logging(command):
    """ Run a command with controlled logging """
    log.debug("Running: %s" % command)
    process = subprocess.Popen(command, stderr=subprocess.PIPE)
    stderr = process.communicate()[1].rstrip()
    process.wait()
    if process.returncode != 0:
        log.error(stderr)
        raise Exception("%s: exit code %d" % (command, process.returncode))
    log.debug(stderr)


@contextmanager
def kubernetes_repo():
    """ Yield a kubernetes repo to copy addons from. """
    repo = "https://github.com/kubernetes/kubernetes.git"
    branch = os.environ["KUBE_VERSION"]
    log.info("Cloning %s with branch %s" % (repo, branch))
    path = tempfile.mkdtemp(prefix="kubernetes")
    try:
        cmd = ["git", "clone", repo, path, "-b", branch, "--depth", "1",
               "--single-branch"]
        run_with_logging(cmd)
        yield path
    finally:
        shutil.rmtree(path)


@contextmanager
def kubernetes_dashboard_repo():
    """ Yield a kubernetes dashboard repo to copy yamls from. """
    repo = "https://github.com/kubernetes/dashboard.git"
    tag = os.environ["KUBE_DASHBOARD_VERSION"]
    log.info("Cloning %s and moving to %s" % (repo, tag))
    path = tempfile.mkdtemp(prefix="dashboard")
    try:
        cmd = ["git", "clone", repo, path, "-b", tag, "--depth", "1",
               "--single-branch"]
        run_with_logging(cmd)
        yield path
    finally:
        shutil.rmtree(path)


@contextmanager
def nvidia_plugin_repo():
    """ Yield a nvidia plugin repo to copy yaml from. """
    repo = "https://github.com/NVIDIA/k8s-device-plugin.git"
    log.info("Cloning %s" % (repo))
    path = tempfile.mkdtemp(prefix="nvidia")
    try:
        cmd = ["git", "clone", repo, path, "--depth", "1"]
        run_with_logging(cmd)
        yield path
    finally:
        shutil.rmtree(path)


@contextmanager
def ceph_csi_repo():
    """ Yield a ceph CSI repo from which to copy template yaml. """
    repo = "http://github.com/ceph/ceph-csi.git"
    log.info("Cloning %s" % (repo))
    path = tempfile.mkdtemp(prefix="ceph")
    try:
        cmd = ["git", "clone", repo, path, "--depth", "1"]
        run_with_logging(cmd)
        yield path
    finally:
        shutil.rmtree(path)


@contextmanager
def cloud_provider_openstack_repo():
    """ Yield a cloud provider openstack repo from which to copy template yaml. """
    repo = "https://github.com/kubernetes/cloud-provider-openstack.git"
    log.info("Cloning %s" % (repo))
    path = tempfile.mkdtemp(prefix="openstack")
    try:
        cmd = ["git", "clone", repo, path, "--depth", "1"]
        run_with_logging(cmd)
        yield path
    finally:
        shutil.rmtree(path)


def add_addon(repo, source, dest, required=True, base='cluster/addons'):
    """ Add an addon template from the given repo and source.

    Any occurrences of 'amd64' are replaced with '{{ arch }}' so the snap can
    fill it in from config. """
    source = os.path.join(repo, base, source)
    if not os.path.exists(source) and not required:
        return
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(source))
    log.debug("Copying: %s -> %s" % (source, dest))
    with open(source, "r") as f:
        content = f.read()
    content = content.replace("amd64", "{{ arch }}")
    content = content.replace("clusterIP: {{ pillar['dns_server'] }}", "# clusterIP: {{ pillar['dns_server'] }}")
    content = content.replace(
		"gcr.io/google_containers/addon-resizer:1.8.1",
		"cdkbot/addon-resizer-{{ arch }}:1.8.1"
    )
    content = content.replace(
		"k8s.gcr.io/addon-resizer:1.8.1",
		"cdkbot/addon-resizer-{{ arch }}:1.8.1"
    )
    # Make sure images come from the configured registry (or use the default)
    content = re.sub(r"image:\s*cdkbot/",
                     "image: {{ registry|default('cdkbot') }}/",
                     content)
    content = re.sub(r"image:\s*k8s.gcr.io/",
                     "image: {{ registry|default('k8s.gcr.io') }}/",
                     content)
    content = re.sub(r"image:\s*nvidia/",
                     "image: {{ registry|default('nvidia') }}/",
                     content)
    with open(dest, "w") as f:
        f.write(content)


def patch_plugin_manifest(repo, file):
    source = os.path.join(repo, file)
    with open(source) as stream:
        manifest = yaml.load(stream)
    manifest['spec']['template']['spec']['containers'][0]['env'] = \
        [{'name': 'DP_DISABLE_HEALTHCHECKS', 'value': 'xids'}]
    manifest['spec']['template']['spec']['nodeSelector'] = {'gpu': 'true'}
    with open(source, 'w') as yaml_file:
        yaml.dump(manifest, yaml_file, default_flow_style=False)


def patch_ceph_secret(repo, file):
    source = os.path.join(repo, file)
    with open(source) as stream:
        manifest = yaml.load(stream)
    manifest['data']['admin'] = "{{ admin_key }}"
    manifest['data']['kubernetes'] = "{{ kubernetes_key }}"
    with open(source, 'w') as yaml_file:
        yaml.dump(manifest, yaml_file, default_flow_style=False)


def patch_ceph_storage_class(repo, file):
    source = os.path.join(repo, file)
    with open(source) as stream:
        manifest = yaml.load(stream)
    manifest['parameters']['monitors'] = "{{ mon_hosts }}"
    manifest['parameters']['pool'] = "{{ pool_name }}"
    manifest['parameters']['fsType'] = "{{ fs_type }}"
    manifest['parameters']['userid'] = "admin"
    manifest['metadata']['name'] = "{{ sc_name }}"
    with open(source, 'w') as yaml_file:
        yaml.dump(manifest, yaml_file, default_flow_style=False)

    # :-/ Would be nice to be able to add this template a different way
    with open(source, "r") as f:
        content = f.read()
    content = content.replace("metadata:", """metadata:
{% if default == true %}
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
{% endif %}""")
    with open(source, "w") as f:
        f.write(content)


def patch_heapster_controller(repo, file):
    source = os.path.join(repo, 'cluster/addons', file)
    with open(source, "r") as f:
        content = f.read()
    # setup kubelet connection to use https port 10250 since 10255 was deprecated due to security concerns
    content = content.replace("kubernetes.summary_api:''", "kubernetes.summary_api:https://kubernetes.default?kubeletPort=10250&kubeletHttps=true")
    with open(source, "w") as f:
        f.write(content)


def patch_metrics_server(repo, file):
    # nuke GKE specific stuff
    source = os.path.join(repo, 'cluster/addons', file)
    with open(source, "r") as f:
        content = f.read()
    content = content.replace("- --kubelet-port=10255", "# - --kubelet-port=10255")
    content = content.replace("- --deprecated-kubelet-completely-insecure=true", "# - --deprecated-kubelet-completely-insecure=true")
    with open(source, "w") as f:
        f.write(content)


def patch_metrics_reader(repo, file):
    # add nodes/stats resource
    source = os.path.join(repo, 'cluster/addons', file)
    with open(source) as stream:
        manifest = list(yaml.load_all(stream))
    for doc in manifest:
        if doc['kind'] == 'ClusterRole':
            for rule in doc['rules']:
                if '' in rule['apiGroups']:
                    rule['resources'].append('nodes/stats')
    with open(source, "w") as yaml_file:
        yaml.dump_all(manifest, yaml_file, default_flow_style=False)


def patch_keystone_deployment(repo, file):
    source = os.path.join(repo, file)
    # :-/ Would be nice to be able to add this template a different way
    with open(source, "r") as f:
        content = f.read()
    content = content.replace('image: k8scloudprovider/k8s-keystone-auth',
                              'image: {{ registry|default("k8scloudprovider") }}/k8s-keystone-auth') # noqa
    content = content.replace("            - --keystone-url",
                              """{% if keystone_ssl_ca %}
            - --keystone-ssl-ca
            - {{ keystone_ssl_ca }}
{% endif %}
            - --keystone-url""")
    with open(source, "w") as f:
        f.write(content)


def patch_dashboard(repo, file):
    source = os.path.join(repo, file)
    with open(source, "r") as f:
        content = f.read()
    content = content.replace("- --auto-generate-certificates",
                              """- --auto-generate-certificates
          - --authentication-mode=basic""")
    with open(source, "w") as f:
        f.write(content)


def get_addon_templates():
    """ Get addon templates. This will clone the kubernetes repo from upstream
    and copy addons to ./templates """
    dest = os.path.abspath("templates")
    os.mkdir(dest)
    with kubernetes_repo() as repo:
        log.info("Copying addons to " + dest)

        add_addon(repo, "dns/kube-dns/kube-dns.yaml.in", dest + "/kube-dns.yaml")

        influxdb = "cluster-monitoring/influxdb"
        add_addon(repo, influxdb + "/grafana-service.yaml", dest)
        patch_heapster_controller(repo, influxdb + "/heapster-controller.yaml")
        add_addon(repo, influxdb + "/heapster-controller.yaml", dest)
        add_addon(repo, influxdb + "/heapster-service.yaml", dest)
        add_addon(repo, influxdb + "/influxdb-grafana-controller.yaml", dest)
        add_addon(repo, influxdb + "/influxdb-service.yaml", dest)

        # Heapster RBAC
        add_addon(repo, "cluster-monitoring/heapster-rbac.yaml", dest)

        # metrics server
        add_addon(repo, "metrics-server/auth-delegator.yaml", dest)
        add_addon(repo, "metrics-server/auth-reader.yaml", dest)
        add_addon(repo, "metrics-server/metrics-apiservice.yaml", dest)
        patch_metrics_server(repo, "metrics-server/metrics-server-deployment.yaml")
        add_addon(repo, "metrics-server/metrics-server-deployment.yaml", dest)
        add_addon(repo, "metrics-server/metrics-server-service.yaml", dest)
        patch_metrics_reader(repo, "metrics-server/resource-reader.yaml")
        add_addon(repo, "metrics-server/resource-reader.yaml", dest)

    with kubernetes_dashboard_repo() as repo:
        log.info("Copying dashboard to " + dest)
        dashboard_yaml = "src/deploy/recommended/kubernetes-dashboard.yaml"
        patch_dashboard(repo, dashboard_yaml)
        add_addon(repo, dashboard_yaml, dest, base='.')

    with nvidia_plugin_repo() as repo:
        log.info("Copying nvidia plugin to " + dest)
        patch_plugin_manifest(repo, "nvidia-device-plugin.yml")
        add_addon(repo, "nvidia-device-plugin.yml", dest, base='.')

    with ceph_csi_repo() as repo:
        log.info("Copying ceph CSI templates to " + dest)
        add_addon(repo, "deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml", dest, base='.')
        add_addon(repo, "deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml", dest, base='.')
        add_addon(repo, "deploy/rbd/kubernetes/csi-rbdplugin.yaml", dest, base='.')

        patch_ceph_secret(repo, "examples/rbd/secret.yaml")
        add_addon(repo, "examples/rbd/secret.yaml", os.path.join(dest, "ceph-secret.yaml"), base='.')

        patch_ceph_storage_class(repo, "examples/rbd/storageclass.yaml")
        add_addon(repo, "examples/rbd/storageclass.yaml", os.path.join(dest, "ceph-storageclass.yaml"), base='.')

        # rbac templates
        add_addon(repo, "deploy/rbd/kubernetes/csi-attacher-rbac.yaml", dest, base='.')
        add_addon(repo, "deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml", dest, base='.')
        add_addon(repo, "deploy/rbd/kubernetes/csi-provisioner-rbac.yaml", dest, base='.')

    with cloud_provider_openstack_repo() as repo:
        log.info("Copying openstack tempates to " + dest)
        add_addon(repo, "examples/webhook/keystone-auth-certs-secret.yaml", dest, base='.')

        patch_keystone_deployment(repo, "examples/webhook/keystone-deployment.yaml")
        add_addon(repo, "examples/webhook/keystone-deployment.yaml", dest, base='.')
        add_addon(repo, "examples/webhook/keystone-service.yaml", dest, base='.')
        add_addon(repo, "examples/webhook/keystone-rbac.yaml", dest, base='.')


def parse_args():
    """ Parse args. This is solely done for the usage output with -h """
    parser = argparse.ArgumentParser(description=description)
    parser.parse_args()


def main():
    """ Parse args and get the addon templates """
    parse_args()
    get_addon_templates()


if __name__ == "__main__":
    main()