Build and upload binaries and Linux packages

.gitignore

@@ -1,2 +1,3 @@
 prometheus-logstash-exporter
-.idea
+/.idea/
+/venv/

.gitlab-ci.yml

@@ -1,36 +1,134 @@
-variables:
-  DOCKER_DRIVER: overlay
-  REPO_NAME: registry.gitlab.com/alxrem/prometheus-logstash-exporter
+stages:
+  - validate
+  - build
+  - upload
+  - announce
 
-build_test_image:
-  variables:
-    TAG: $CI_COMMIT_REF_SLUG
-  image: docker:latest
-  services:
-  - docker:dind
-  stage: build
+.release:
   only:
-  - branches
+    - tags
   except:
-  - master
-  when: manual
+    - master
+
+.build_package:
+  stage: build
+  extends:
+    - .release
   script:
-  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
-  - docker build -t $REPO_NAME:$TAG .
-  - docker push $REPO_NAME:$TAG
+    - ci/build_package
+  artifacts:
+    paths:
+      - .result/
+    expire_in: 30m
 
-build_release_image:
-  variables:
-    TAG: $CI_COMMIT_TAG
+.upload_package:
+  stage: upload
+  extends:
+    - .release
+  image: registry.gitlab.com/alxrem/package_cloud
+  script:
+    - ci/upload_package
+
+validate:
+  stage: validate
+  image: alpine
+  extends:
+    - .release
+  script:
+    - apk -U add git
+    - ci/validate_version
+
+build:image:
+  stage: build
   image: docker:latest
   services:
-  - docker:dind
+    - docker:19.03.5-dind
+  except:
+    - master
+  script:
+    - ci/build_image
+
+build:stretch:
+  extends:
+    - .build_package
+  image: debian:stretch
+
+upload:stretch:
+  variables:
+    DIST: debian/stretch
+  extends:
+    - .upload_package
+  needs:
+    - build:stretch
+
+build:buster:
+  extends:
+    - .build_package
+  image: debian:buster
+
+upload:buster:
+  variables:
+    DIST: debian/buster
+  extends:
+    - .upload_package
+  needs:
+    - build:buster
+
+build:xenial:
+  extends:
+    - .build_package
+  image: ubuntu:xenial
+
+upload:xenial:
+  variables:
+    DIST: ubuntu/xenial
+  extends:
+    - .upload_package
+  needs:
+    - build:xenial
+
+build:bionic:
+  extends:
+    - .build_package
+  image: ubuntu:bionic
+
+upload:bionic:
+  variables:
+    DIST: ubuntu/bionic
+  extends:
+    - .upload_package
+  needs:
+    - build:bionic
+
+build:binaries:
   stage: build
-  only:
-  - tags
+  extends:
+    - .release
+  image: golang:alpine
+  script:
+    - apk add -U binutils
+    - ci/build_binaries
+  artifacts:
+    paths:
+      - binaries/
+    expire_in: 30m
+
+announce:
+  stage: announce
+  extends:
+    - .release
+  image: python:3-alpine
+  needs:
+    - build:binaries
+    - job: upload:stretch
+      artifacts: false
+    - job: upload:buster
+      artifacts: false
+    - job: upload:xenial
+      artifacts: false
+    - job: upload:bionic
+      artifacts: false
   script:
-  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
-  - docker build -t $REPO_NAME:$TAG .
-  - docker tag $REPO_NAME:$TAG $REPO_NAME:latest
-  - docker push $REPO_NAME:$TAG
-  - docker push $REPO_NAME:latest
+    - pip install requests
+    - apk -U add git
+    - ci/gitlab_release

README.md

@@ -18,10 +18,12 @@ To see all available configuration flags:
 Packages
 --------
 
-Packages for Debian Stretch and Ubuntu Xenial are available on
+Binary builds are available on the [releases page of Gitlab project](https://gitlab.com/alxrem/prometheus-logstash-exporter/-/releases).
+
+Packages for latest Debian and Ubuntu releases are available on
 [PackageCloud](https://packagecloud.io/alxrem/prometheus-logstash-exporter/).
 
-Docker image is available at [Docker Hub](https://hub.docker.com/r/alxrem/prometheus-logstash-exporter/).
+Docker images are available at [Docker Hub](https://hub.docker.com/r/alxrem/prometheus-logstash-exporter/).
 Pull the latest version with
 
     docker pull alxrem/prometheus-logstash-exporter

ci/build_binaries

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+BUILDDIR=/go/src/gitlab.com/$CI_PROJECT_PATH
+DEST=`pwd`/binaries
+GOARCH=amd64
+
+mkdir -p ${DEST}/
+mkdir -p `dirname $BUILDDIR`
+ln -s `pwd` $BUILDDIR
+cd $BUILDDIR
+
+for GOOS in windows linux darwin; do
+  BINARY="${DEST}/${CI_PROJECT_NAME}-${CI_COMMIT_TAG}-${GOOS}-${GOARCH}"
+  SUM="${BINARY}.sha256"
+  CGO_ENABLED=0 go build -o $BINARY
+  strip $BINARY
+  sha256sum $BINARY | sed -e 's|\(\w\+\s\+\).*/\(.*\)$|\1\2|' > $SUM
+done

ci/build_image

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+TAG=${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}
+docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+docker build -t $CI_REGISTRY_IMAGE:$TAG .
+docker push $REPO_NAME:$TAG
+
+if [ -n "$CI_COMMIT_TAG" ]; then
+  docker tag $CI_REGISTRY_IMAGE:$TAG $CI_REGISTRY_IMAGE:latest
+  docker push $CI_REGISTRY_IMAGE:latest
+
+  docker login -u $DOCKERHUB_USER -p $DOCKERHUB_TOKEN
+  docker tag $CI_REGISTRY_IMAGE:$TAG alxrem/prometheus-logstash-exporter:$TAG
+  docker push alxrem/prometheus-logstash-exporter:$TAG
+
+  docker tag $CI_REGISTRY_IMAGE:$TAG alxrem/prometheus-logstash-exporter:latest
+  docker push alxrem/prometheus-logstash-exporter:latest
+fi

ci/build_package

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+apt-get -qq update
+apt-get -qq --no-install-recommends install devscripts equivs
+
+mk-build-deps -irBt "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -qq -y"
+dpkg-buildpackage -uc -us
+rm -rf .result && mkdir -p .result
+find ../ -maxdepth 1 -type f -exec mv '{}' .result/ ';'
+find .result/

ci/gitlab_release

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+from glob import glob
+from urllib.parse import urljoin
+
+import requests
+
+
+def do_release():
+    gitlab_token = os.environ["GITLAB_TOKEN"]
+    version = os.environ["CI_COMMIT_TAG"]
+    project = os.environ["CI_PROJECT_ID"]
+    project_url = os.environ["CI_PROJECT_URL"]
+    headers = {"PRIVATE-TOKEN": gitlab_token}
+
+    # https://docs.gitlab.com/ee/api/projects.html#upload-a-file
+    uploads_url = \
+        "https://gitlab.com/api/v4/projects/{}/uploads".format(project)
+
+    # https://docs.gitlab.com/ee/api/releases/#create-a-release
+    release_url = \
+        "https://gitlab.com/api/v4/projects/{}/releases".format(project)
+
+    binaries = []
+    assets_links = []
+    for binary in glob("binaries/*"):
+        files = {"file": open(binary, "rb")}
+
+        r = requests.post(uploads_url, headers=headers, files=files)
+        if r.status_code >= 400:
+            print(r.text)
+            exit(1)
+
+        upload = r.json()
+        binaries.append(upload["markdown"])
+        assets_links.append({
+            'name': upload['alt'],
+            'url': urljoin(project_url + '/', upload['url'].lstrip('/')),
+        })
+
+    description = \
+        subprocess.getoutput("git tag {} -l --format='%(contents:body)'"
+                             .format(version))
+
+    release = {
+        "name": version,
+        "tag_name": version,
+        "description": description,
+        'assets': {
+            'links': sorted(assets_links, key=lambda x: x['name'], reverse=True),
+        }
+    }
+    r = requests.post(release_url, headers=headers, json=release)
+    if r.status_code >= 400:
+        print(r.text)
+        exit(1)
+
+
+if __name__ == "__main__":
+    do_release()

ci/upload_package

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+package_cloud push alxrem/prometheus-logstash-exporter/$DIST .result/*.deb

ci/validate_version

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+SANITIZED_VERSION=$(echo "$CI_COMMIT_TAG" | egrep -ox '[0-9]+\.[0-9]+\.[0-9]+.*')
+
+if [ -z "$SANITIZED_VERSION" ]; then
+  echo "Invalid version format \"$CI_COMMIT_TAG\""
+  exit 1
+fi
+
+ANNOTATION=$(git tag $CI_COMMIT_TAG -l --format='%(contents:body)' | tr -d '[[:space:]]')
+if [ -z "$ANNOTATION" ]; then
+  echo "Tag should by annotated"
+  exit 1
+fi
+
+CHANGELOG_EXPECTED="prometheus-logstash-exporter ($CI_COMMIT_TAG)"
+CHANGELOG_FOUND=$(head -n1 debian/changelog | fgrep -o "$CHANGELOG_EXPECTED")
+
+if [ "$CHANGELOG_FOUND" != "$CHANGELOG_EXPECTED" ]; then
+  echo "Version $CHANGELOG_EXPECTED expected"
+  echo
+  echo "Found $(head -n1 debian/changelog)"
+  exit 1
+fi
+
+exit 0

debian/changelog

@@ -0,0 +1,32 @@
+prometheus-logstash-exporter (0.6.1) unstable; urgency=medium
+
+  * Technical release to tag updated CI scripts.
+
+ -- Alexey Remizov <[email protected]>  Thu, 10 Mar 2020 01:23:50 +0300
+
+prometheus-logstash-exporter (0.5.0) unstable; urgency=medium
+
+  * Parse values of type timestamp
+    Thanks to Michael DOUBEZ
+
+ -- Alexey Remizov <[email protected]>  Thu, 15 Aug 2019 00:57:47 +0300
+
+prometheus-logstash-exporter (0.4.0) unstable; urgency=medium
+
+  * Fixed processing of `filters.patterns_per_field` section
+    Thanks to Casey Weed <[email protected]>
+
+ -- Alexey Remizov <[email protected]>  Mon, 20 Aug 2018 23:11:00 +0300
+
+prometheus-logstash-exporter (0.2.0) unstable; urgency=medium
+
+  * Added support of Logstash 6.x.
+    Thanks to Guo Xiang Tan <[email protected]>
+
+ -- Alexey Remizov <[email protected]>  Thu, 14 Dec 2017 02:29:59 +0300
+
+prometheus-logstash-exporter (0.1.0) unstable; urgency=low
+
+  * Initial release.
+
+ -- Alexey Remizov <[email protected]>  Tue, 12 Sep 2017 00:49:07 +0300

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,15 @@
+Source: prometheus-logstash-exporter
+Maintainer: Alexey Remizov <[email protected]>
+Section: utils
+Priority: optional
+Build-Depends: debhelper (>= 9), dh-systemd, dh-golang, golang (>= 2:1.6)
+Standards-Version: 3.9.4
+
+Package: prometheus-logstash-exporter
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Suggests: logstash
+Description: Prometheus exporter for Logstash metrics
+ Prometheus exporter for metrics provided by Node Stats API of Logstash.
+
+