Merge pull request #581 from Mirantis/ivan4th/release-automation

Implement release automation

Author: lukaszo

.circleci/config.yml

@@ -333,6 +333,25 @@ jobs:
   push_tested:
     <<: *push_images
 
+  release:
+    <<: *defaults
+    steps:
+    - run:
+        <<: *prereqs
+    - checkout
+    - setup_remote_docker
+    - run:
+        <<: *setup_env
+    - restore_cache:
+        <<: *restore_image_cache
+    - run:
+        <<: *restore_images_and_vendor
+    - run:
+        name: Make a release
+        command: |
+          build/portforward.sh 18730&
+          build/cmd.sh release "${CIRCLE_TAG}"
+
 workflows:
   version: 2
   build-and-test:
@@ -416,3 +435,12 @@ workflows:
           tags:
             only:
               - /^v[0-9].*/
+    - release:
+        requires:
+        - push_tested
+        filters:
+          branches:
+            ignore: /.*/
+          tags:
+            only:
+              - /^v[0-9].*/

build/cmd.sh

@@ -41,6 +41,13 @@ containers_run='.items[0].spec.template.spec.containers|=map(.volumeMounts|=map(
 initContainers_run='.items[0].spec.template.spec.initContainers|=map(.volumeMounts|=map(select(.name!="run"))+[{"mountPath": "/run:shared", "name": "run"}])'
 containers_k8s_dir='.items[0].spec.template.spec.containers[1].volumeMounts|=map(select(.name!="k8s-pods-dir"))+[{"mountPath": "/var/lib/kubelet/pods:shared", "name": "k8s-pods-dir"}]'
 
+function image_tags_filter {
+    local tag="${1}"
+    local prefix=".items[0].spec.template.spec."
+    local suffix="|=map(.image=\"mirantis/virtlet:${tag}\")"
+    echo -n "${prefix}containers${suffix}|${prefix}initContainers${suffix}"
+}
+
 # from build/common.sh in k8s
 function rsync_probe {
     # Wait unil rsync is up and running.
@@ -160,6 +167,7 @@ function ensure_build_container {
                -e CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-}" \
                -e CIRCLE_BRANCH="${CIRCLE_PULL_REQUEST:-}" \
                -e VIRTLET_ON_MASTER="${VIRTLET_ON_MASTER:-}" \
+               -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
                ${docker_cert_args[@]+"${docker_cert_args[@]}"} \
                --name virtlet-build \
                --tmpfs /tmp \
@@ -348,9 +356,6 @@ function build_image_internal {
 }
 
 function install_vendor_internal {
-    if ! hash glide 2>/dev/null; then
-        go get github.com/Masterminds/glide
-    fi
     if [ ! -d vendor ]; then
         glide install --strip-vendor
     fi
@@ -375,6 +380,47 @@ function build_internal {
     go test -i -c -o "${project_dir}/_output/virtlet-e2e-tests" ./tests/e2e
 }
 
+function patch_yaml {
+    local kubectl="${1}"
+    local filter="${2}"
+    "${kubectl}" convert --local -o json -f deploy/virtlet-ds.yaml |
+        jq "${filter}" |
+        "${kubectl}" convert --local -o yaml -f -
+}
+
+function release_internal {
+    local tag="${1}"
+    local -a opts=(--user Mirantis --repo virtlet --tag "${tag}")
+    local pre_release=
+    # TODO: uncomment this 'if/fi' once we start making 'non-pre' releases
+    # if [[ ${tag} =~ -(test|pre).*$ ]]; then
+    pre_release="--pre-release"
+    # fi
+    if github-release --quiet delete "${opts[@]}"; then
+        echo >&2 "Replacing the old Virtlet release"
+    fi
+    github-release release "${opts[@]}" \
+                   --name "$(git tag -l --format='%(contents:subject)' "${tag}")" \
+                   --description "$(git tag -l --format='%(contents:body)' "${tag}")" \
+                   ${pre_release}
+    # piping the patched yaml directly to
+    # 'github-release upload ... --file -' causes GH API error:
+    # "error: could not upload, status code (400 Bad Request), msg: Bad Content-Length: , errors:"
+    mkdir -p _output
+    patch_yaml kubectl "$(image_tags_filter "${tag}")" >_output/virtlet-ds.yaml
+    github-release upload "${opts[@]}" \
+                   --name virtlet-ds.yaml \
+                   --replace \
+                   --file _output/virtlet-ds.yaml
+    # use older kubectl for 1.7 to avoid making yaml
+    # that can't be consumed by k8s 1.7
+    patch_yaml kubectl.old "$(image_tags_filter "${tag}")|${containers_run}|${initContainers_run}|${containers_k8s_dir}" >_output/virtlet-ds-1.7.yaml
+    github-release upload "${opts[@]}" \
+                   --name virtlet-ds-1.7.yaml \
+                   --replace \
+                   --file _output/virtlet-ds-1.7.yaml
+}
+
 function e2e {
     ensure_build_container
     local cluster_url
@@ -395,6 +441,7 @@ function usage {
     echo >&2 "  $0 gotest [TEST_ARGS...]"
     echo >&2 "  $0 gobuild [BUILD_ARGS...]"
     echo >&2 "  $0 run CMD..."
+    echo >&2 "  $0 release TAG"
     exit 1
 }
 
@@ -470,6 +517,15 @@ case "${cmd}" in
     start-build-container)
         ensure_build_container
         ;;
+    release)
+        if [[ ! ${1:-} ]]; then
+            echo >&2 "must specify the tag"
+        fi
+        ( vcmd "build/cmd.sh release-internal '${1}'" )
+        ;;
+    release-internal)
+        release_internal "$@"
+        ;;
     *)
         usage
         ;;

images/Dockerfile.build

@@ -1,7 +1,7 @@
 # TODO: generate this tag. unfortunately can't use ARG:
 # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
 # (but add a note about it here for the future)
-FROM mirantis/virtlet-build:v1-b6890e358ebebdad0ce449322a3d2646
+FROM mirantis/virtlet-build:v1-d9d67ed7fd827238a8f604db7e3baeaa
 MAINTAINER Ivan Shvedunov <[email protected]>
 
 LABEL virtlet.image="build"

images/Dockerfile.build-base

@@ -1,16 +1,18 @@
 # TODO: generate this tag. unfortunately can't use ARG:
 # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
 # (but add a note about it here for the future)
-FROM mirantis/virtlet-base:v1-13deb720a48ab153ce8bbe95840af75e
+FROM mirantis/virtlet-base:v1-22de6b4e88a489a2c0b441e3f3d01a3e
 MAINTAINER Ivan Shvedunov <[email protected]>
 
 LABEL virtlet.image="build-base"
 
 ENV DOCKER_CLIENT_VERSION 1.12.6
 ENV DOCKER_CLIENT_DOWNLOAD_URL_BASE https://get.docker.com/builds
 ENV DOCKER_CLIENT_SHA256 cce8aa52a33f007673b9c7ed0cb786359aa6f3f8a9875aaf69799c71f86e8a43
-ENV KUBECTL_VERSION v1.8.4
-ENV KUBECTL_SHA1 8e2314db816b9b4465c5f713c1152cb0603db15e
+ENV KUBECTL_VERSION v1.8.8
+ENV KUBECTL_SHA1 f6bc48e70202f168bd71e81dd1504686890cd5ed
+ENV OLD_KUBECTL_VERSION v1.7.12
+ENV OLD_KUBECTL_SHA1 385229d4189e4f7978de42f237d6c443c0534edd
 
 # Go installation commands are based on official Go container:
 # https://github.com/docker-library/golang/blob/18ee81a2ec649dd7b3d5126b24eef86bc9c86d80/1.7/Dockerfile
@@ -35,11 +37,13 @@ RUN apt-get update && \
                 dhcpcd5 \
                 tcpdump \
                 iputils-ping \
-                mercurial && \
+                mercurial \
+                jq && \
     apt-get clean
 
 # Install docker client
-RUN curl -sSL "${DOCKER_CLIENT_DOWNLOAD_URL_BASE}/$(uname -s)/$(uname -m)/docker-${DOCKER_CLIENT_VERSION}.tgz" | \
+RUN mkdir -p /usr/local/bin && \
+    curl -sSL "${DOCKER_CLIENT_DOWNLOAD_URL_BASE}/$(uname -s)/$(uname -m)/docker-${DOCKER_CLIENT_VERSION}.tgz" | \
     tar -C /usr/local/bin -xz --strip-components=1 docker/docker && \
     echo "${DOCKER_CLIENT_SHA256}  /usr/local/bin/docker" | sha256sum -c -
 
@@ -55,11 +59,14 @@ ENV PATH /go/src/github.com/Mirantis/virtlet/_output:$GOPATH/bin:/usr/local/go/b
 RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 WORKDIR $GOPATH
 
-# Install kubectl
-RUN curl -L https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl
-RUN echo "${KUBECTL_SHA1}  /usr/local/bin/kubectl" |sha1sum -c
-RUN mkdir -p /usr/local/bin
-RUN chmod +x /usr/local/bin/kubectl
+# Install kubectl.
+# Also install old kubectl for v1.7 (needed to make the release yaml)
+RUN curl -L https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl && \
+    echo "${KUBECTL_SHA1}  /usr/local/bin/kubectl" | sha1sum -c && \
+    chmod +x /usr/local/bin/kubectl && \
+    curl -L https://storage.googleapis.com/kubernetes-release/release/${OLD_KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl.old && \
+    echo "${OLD_KUBECTL_SHA1}  /usr/local/bin/kubectl.old" | sha1sum -c && \
+    chmod +x /usr/local/bin/kubectl.old
 
 # TODO: update CNI ver
 RUN mkdir -p ~/.ssh && \
@@ -73,3 +80,8 @@ RUN mkdir -p ~/.ssh && \
 RUN mkdir -p /images && \
     curl -L https://github.com/Mirantis/virtlet/releases/download/v0.8.2/cirros.img -o /images/cirros.img && \
     echo "${CIRROS_SHA256}  /images/cirros.img" | sha256sum -c -
+
+# Install Go tooling
+RUN go get github.com/aktau/github-release && \
+    go get github.com/Masterminds/glide
+

images/Dockerfile.virtlet

@@ -1,7 +1,7 @@
 # TODO: generate this tag. unfortunately can't use ARG:
 # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
 # (but add a note about it here for the future)
-FROM mirantis/virtlet-base:v1-13deb720a48ab153ce8bbe95840af75e
+FROM mirantis/virtlet-base:v1-22de6b4e88a489a2c0b441e3f3d01a3e
 MAINTAINER Ivan Shvedunov <[email protected]>
 
 LABEL virtlet.image="virtlet"

images/Dockerfile.virtlet-base

@@ -12,3 +12,7 @@ RUN apt-get update && \
                        netbase iproute2 iptables ebtables vncsnapshot \
                        socat && \
     apt-get clean
+
+# 19 Feb 2018: bump: libvirt version problem
+# The following packages have unmet dependencies:
+#  libvirt-bin : Breaks: qemu-kvm (< 1:2.5+dfsg-5ubuntu10.20~) but 1:2.5+dfsg-5ubuntu10.16 is to be installed