Merge branch 'master' into feat/upstream-jobs

.github/workflows/build.yml

@@ -5,10 +5,16 @@ permissions: read-all
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - docker-full
+          - docker-minimal
     steps:
       - name: Checkout code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
       - name: Build Container
-        run: make docker
+        run: make ${{matrix.target}}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
           - { name: k8s, on: ubuntu-latest }
           - { name: datasources, on: ubuntu-latest }
           - { name: opensearch, on: ubuntu-latest }
-          - { name: elasticsearch, on: ubuntu-latest }
+          # - { name: elasticsearch, on: ubuntu-latest }
           - { name: git, on: ubuntu-latest }
           # - restic
     runs-on: ${{ matrix.suite.on }}

Makefile

@@ -82,15 +82,18 @@ generate: .bin/controller-gen
 	.bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./api/..."
 
 # Build the docker image
-docker:
-	docker build . -f build/full/Dockerfile -t ${IMG_F}
+docker: docker-minimal docker-full
+
+docker-full:
+	docker build . -f build/full/Dockerfile -t ${IMG}
+
+docker-minimal:
 	docker build . -f build/minimal/Dockerfile -t ${IMG}
 
 # Build the docker image
 docker-dev: linux
 	docker build ./ -f build/dev/Dockerfile -t ${IMG}
 
-
 docker-push-%:
 	docker build . -f build/full/Dockerfile -t ${IMG_F}
 	docker build . -f build/minimal/Dockerfile -t ${IMG}

api/v1/checks.go

@@ -74,7 +74,7 @@ type HTTPCheck struct {
 	ResponseCodes []int `yaml:"responseCodes,omitempty" json:"responseCodes,omitempty"`
 	// Exact response content expected to be returned by the endpoint.
 	ResponseContent string `yaml:"responseContent,omitempty" json:"responseContent,omitempty"`
-	// Path and value to of expect JSON response by the endpoint
+	// Deprecated, use expr and jsonpath function
 	ResponseJSONContent *JSONCheck `yaml:"responseJSONContent,omitempty" json:"responseJSONContent,omitempty"`
 	// Maximum number of days until the SSL Certificate expires.
 	MaxSSLExpiry int `yaml:"maxSSLExpiry,omitempty" json:"maxSSLExpiry,omitempty"`

build/full/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder
+FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder
 WORKDIR /app
 
 ARG NAME
@@ -12,27 +12,37 @@ RUN go mod download
 COPY ./ ./
 RUN make build
 
-FROM eclipse-temurin:11.0.18_10-jdk-focal@sha256:509043cc38d37a5bd44720b471c38bef40fb34de67c03baaa67a5a9d8cda52a0
+FROM eclipse-temurin:11.0.20.1_1-jdk-jammy@sha256:1584fd589b45a67b6f56b0c702776ca3d5640d1001f7848f5bcd19cb10545eaa
 WORKDIR /app
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-  apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \
+  apt-get install -y curl unzip ca-certificates jq tzdata wget gnupg2 bzip2 apt-transport-https lsb-release python3 python3-pip --no-install-recommends && \
   rm -Rf /var/lib/apt/lists/*  && \
   rm -Rf /usr/share/doc && rm -Rf /usr/share/man  && \
   apt-get clean
 
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-  echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
-  apt-get update && apt-get install -y \
-  google-chrome-stable \
-  fontconfig \
-  fonts-ipafont-gothic \
-  fonts-wqy-zenhei \
-  fonts-thai-tlwg \
-  fonts-kacst \
-  fonts-symbola \
-  fonts-noto \
-  fonts-freefont-ttf \
-  --no-install-recommends
+RUN pip3 install  pip pyyaml lxml requests robotframework \
+    robotframework \
+    robotframework-jsonlibrary \
+    robotframework-jsonschemalibrary \
+    robotframework-requests \
+    robotframework-restlibrary \
+    robotframework-seleniumlibrary \
+    robotframework-excellib \
+    robotframework-crypto \
+    robotframework-databaselibrary \
+    psycopg2-binary \
+    PyMySQL
+
+RUN mkdir -p /etc/apt/keyrings && \
+  curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \
+    gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \
+  chmod go+r /etc/apt/keyrings/microsoft.gpg &&  \
+  echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \
+  cat /etc/apt/sources.list.d/azure-cli.list && \
+  apt-get update && \
+  apt-get install -y azure-cli && \
+  apt-get clean
 
 RUN apt-get update && apt-get upgrade -y && \
   rm -Rf /var/lib/apt/lists/* && \
@@ -45,42 +55,37 @@ RUN curl -L https://github.com/restic/restic/releases/download/v${RESTIC_VERSION
   mv /app/restic /usr/local/bin/ && \
   rm -rf /app/restic.bz2
 
-ENV JMETER_VERSION=5.5
+ENV JMETER_VERSION=5.6.2
 RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip -o apache-jmeter-${JMETER_VERSION}.zip && \
-  unzip apache-jmeter-${JMETER_VERSION}.zip -d /opt && \
+  unzip -q apache-jmeter-${JMETER_VERSION}.zip -d /opt && \
   rm apache-jmeter-${JMETER_VERSION}.zip
 
 ENV PATH /opt/apache-jmeter-${JMETER_VERSION}/bin/:$PATH
 
-RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz && \
-  tar xf askgit.tar.gz && \
-  mv askgit /usr/local/bin/askgit && \
-  rm askgit.tar.gz && \
-  wget http://mirrors.kernel.org/ubuntu/pool/main/o/openssl/openssl_1.1.1f-1ubuntu2.19_amd64.deb && \
-  dpkg -i openssl_1.1.1f-1ubuntu2.19_amd64.deb && \
-  rm openssl_1.1.1f-1ubuntu2.19_amd64.deb
+
+RUN curl -L  https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-linux-amd64.tar.gz -o mergestat.tar.gz && \
+  tar zxf mergestat.tar.gz -C /usr/local/bin/ && \
+  rm mergestat.tar.gz
 
 # The best developer experience for load testing
-ENV K6_VERSION=v0.44.0
+ENV K6_VERSION=v0.47.0
 RUN curl -L https://github.com/grafana/k6/releases/download/${K6_VERSION}/k6-${K6_VERSION}-linux-amd64.tar.gz -o k6.tar.gz && \
   tar xvf k6.tar.gz && \
   mv k6-${K6_VERSION}-linux-amd64/k6 /usr/local/bin/k6 && \
   rm k6.tar.gz
 
 # Benthos is a high performance and resilient stream processor
-RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.15.0
+RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.22.0
 
 # Commandline tool for running SQL queries against JSON, CSV, Excel, Parquet, and more
 RUN curl -L https://github.com/multiprocessio/dsq/releases/download/v0.23.0/dsq-linux-x64-v0.23.0.zip -o dsq.zip && \
-  unzip dsq.zip && \
+  unzip -q dsq.zip && \
   mv dsq /usr/local/bin/dsq && \
   rm dsq.zip
 
 # Install alexellis/arkade as root
 RUN curl -sLS https://get.arkade.dev | sh
 
-# Install Azure CLI (need to install as root)
-RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
 
 RUN mkdir /opt/database && groupadd --gid 1000 canary && \
   useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \
@@ -95,14 +100,15 @@ ENV PATH="${PATH}:/var/lib/canary/bin/"
 
 # Install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-  unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
+  unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
   rm awscliv2.zip
 
 # Install GCP CLI
-RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
-  tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
+ENV GCLOUD_VERSION=441.0.0
+RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+  tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
   ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \
-  rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz
+  rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz
 
 COPY --from=builder /app/.bin/canary-checker /app
 

build/minimal/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder
+FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder
 WORKDIR /app
 
 ARG NAME
@@ -12,22 +12,30 @@ RUN go mod download
 COPY ./ ./
 RUN make build
 
-FROM ubuntu 
+FROM ubuntu:jammy-20231004@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f
 WORKDIR /app
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-  apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \
-  rm -Rf /var/lib/apt/lists/*  && \
-  rm -Rf /usr/share/doc && rm -Rf /usr/share/man  && \
+  apt-get install -y curl unzip ca-certificates jq  tzdata wget gnupg2 bzip2 apt-transport-https lsb-release --no-install-recommends  && \
   apt-get clean
 
-COPY --from=builder /app/.bin/canary-checker /app
+RUN mkdir -p /etc/apt/keyrings && \
+  curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \
+    gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \
+  chmod go+r /etc/apt/keyrings/microsoft.gpg &&  \
+  echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \
+  cat /etc/apt/sources.list.d/azure-cli.list && \
+  apt-get update && \
+  apt-get install -y azure-cli && \
+  apt-get clean
+
+RUN apt-get update && apt-get upgrade -y && \
+  rm -Rf /var/lib/apt/lists/* && \
+  apt-get clean
 
 # Install alexellis/arkade as root
 RUN curl -sLS https://get.arkade.dev | sh
 
-# Install Azure CLI (need to install as root)
-RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-
 RUN mkdir /opt/database && groupadd --gid 1000 canary && \
   useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \
   chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app
@@ -41,15 +49,17 @@ ENV PATH="${PATH}:/var/lib/canary/bin/"
 
 # Install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-  unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
+  unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
   rm awscliv2.zip
 
 # Install GCP CLI
-RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
-  tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
+ENV GCLOUD_VERSION=441.0.0
+RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+  tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
   ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \
-  rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz
+  rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz
 
-RUN /app/canary-checker go-offline
+COPY --from=builder /app/.bin/canary-checker /app
 
+RUN /app/canary-checker go-offline
 ENTRYPOINT ["/app/canary-checker"]

checks/github.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	osExec "os/exec"
-	"strings"
 
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
@@ -49,27 +48,23 @@ func (c *GitHubChecker) Check(ctx *context.Context, extConfig external.Check) pk
 		}
 	}
 
-	askGitCmd := fmt.Sprintf("GITHUB_TOKEN=%v askgit \"%v\" --format json", githubToken, check.Query)
+	askGitCmd := fmt.Sprintf("mergestat \"%v\" --format json", check.Query)
+	if ctx.IsTrace() {
+		ctx.Tracef("Executing askgit command: %v", askGitCmd)
+	}
 	cmd := osExec.Command("bash", "-c", askGitCmd)
+	cmd.Env = append(cmd.Env, "GITHUB_TOKEN="+githubToken)
 	output, err := cmd.CombinedOutput()
 	if err != nil {
 		return results.Failf("error executing askgit command. output=%q: %v", output, err)
 	}
 
-	rows := string(output)
-	var rowResults = make([]map[string]string, 0)
-	for _, row := range strings.Split(rows, "\n") {
-		if row == "" {
-			continue
-		}
-		var rowResult map[string]string
-		err := json.Unmarshal([]byte(row), &rowResult)
-		if err != nil {
-			return results.Failf("error parsing askgit result: %v", err)
-		}
-
-		rowResults = append(rowResults, rowResult)
+	var rowResults = make([]map[string]any, 0)
+	err = json.Unmarshal(output, &rowResults)
+	if err != nil {
+		return results.Failf("error parsing mergestat result: %v", err)
 	}
+
 	result.AddDetails(rowResults)
 	return results
 }