From 32f8f19f703468b0563bb20c7eb67e2d5b62d1b1 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 18 Oct 2023 00:33:10 +0300 Subject: [PATCH] chore: bump images, mergestat and install robot --- .github/workflows/build.yml | 8 ++- .github/workflows/test.yml | 2 +- Makefile | 9 ++- build/full/Dockerfile | 74 ++++++++++++---------- build/minimal/Dockerfile | 38 +++++++---- checks/github.go | 25 +++----- fixtures/git/_setup.sh | 22 +++---- fixtures/git/git_check_pass.yaml | 4 +- fixtures/git/git_test_expression_pass.yaml | 2 +- 9 files changed, 100 insertions(+), 84 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f7228191..f147b6909 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,10 +5,16 @@ permissions: read-all jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + - docker-full + - docker-minimal steps: - name: Checkout code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Build Container - run: make docker + run: make ${{matrix.target}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 362759c38..d44740a30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: - { name: k8s, on: ubuntu-latest } - { name: datasources, on: ubuntu-latest } - { name: opensearch, on: ubuntu-latest } - - { name: elasticsearch, on: ubuntu-latest } + # - { name: elasticsearch, on: ubuntu-latest } - { name: git, on: ubuntu-latest } # - restic runs-on: ${{ matrix.suite.on }} diff --git a/Makefile b/Makefile index 0352dda17..0e834f885 100644 --- a/Makefile +++ b/Makefile @@ -82,15 +82,18 @@ generate: .bin/controller-gen .bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./api/..." # Build the docker image -docker: - docker build . -f build/full/Dockerfile -t ${IMG_F} +docker: docker-minimal docker-full + +docker-full: + docker build . -f build/full/Dockerfile -t ${IMG} + +docker-minimal: docker build . -f build/minimal/Dockerfile -t ${IMG} # Build the docker image docker-dev: linux docker build ./ -f build/dev/Dockerfile -t ${IMG} - docker-push-%: docker build . -f build/full/Dockerfile -t ${IMG_F} docker build . -f build/minimal/Dockerfile -t ${IMG} diff --git a/build/full/Dockerfile b/build/full/Dockerfile index e6081edb9..07258e4b8 100644 --- a/build/full/Dockerfile +++ b/build/full/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder +FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder WORKDIR /app ARG NAME @@ -12,27 +12,37 @@ RUN go mod download COPY ./ ./ RUN make build -FROM eclipse-temurin:11.0.18_10-jdk-focal@sha256:509043cc38d37a5bd44720b471c38bef40fb34de67c03baaa67a5a9d8cda52a0 +FROM eclipse-temurin:11.0.20.1_1-jdk-jammy@sha256:1584fd589b45a67b6f56b0c702776ca3d5640d1001f7848f5bcd19cb10545eaa WORKDIR /app +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \ + apt-get install -y curl unzip ca-certificates jq tzdata wget gnupg2 bzip2 apt-transport-https lsb-release python3 python3-pip --no-install-recommends && \ rm -Rf /var/lib/apt/lists/* && \ rm -Rf /usr/share/doc && rm -Rf /usr/share/man && \ apt-get clean -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \ - apt-get update && apt-get install -y \ - google-chrome-stable \ - fontconfig \ - fonts-ipafont-gothic \ - fonts-wqy-zenhei \ - fonts-thai-tlwg \ - fonts-kacst \ - fonts-symbola \ - fonts-noto \ - fonts-freefont-ttf \ - --no-install-recommends +RUN pip3 install pip pyyaml lxml requests robotframework \ + robotframework \ + robotframework-jsonlibrary \ + robotframework-jsonschemalibrary \ + robotframework-requests \ + robotframework-restlibrary \ + robotframework-seleniumlibrary \ + robotframework-excellib \ + robotframework-crypto \ + robotframework-databaselibrary \ + psycopg2-binary \ + PyMySQL + +RUN mkdir -p /etc/apt/keyrings && \ + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \ + gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \ + chmod go+r /etc/apt/keyrings/microsoft.gpg && \ + echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \ + cat /etc/apt/sources.list.d/azure-cli.list && \ + apt-get update && \ + apt-get install -y azure-cli && \ + apt-get clean RUN apt-get update && apt-get upgrade -y && \ rm -Rf /var/lib/apt/lists/* && \ @@ -45,42 +55,37 @@ RUN curl -L https://github.com/restic/restic/releases/download/v${RESTIC_VERSION mv /app/restic /usr/local/bin/ && \ rm -rf /app/restic.bz2 -ENV JMETER_VERSION=5.5 +ENV JMETER_VERSION=5.6.2 RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip -o apache-jmeter-${JMETER_VERSION}.zip && \ - unzip apache-jmeter-${JMETER_VERSION}.zip -d /opt && \ + unzip -q apache-jmeter-${JMETER_VERSION}.zip -d /opt && \ rm apache-jmeter-${JMETER_VERSION}.zip ENV PATH /opt/apache-jmeter-${JMETER_VERSION}/bin/:$PATH -RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz && \ - tar xf askgit.tar.gz && \ - mv askgit /usr/local/bin/askgit && \ - rm askgit.tar.gz && \ - wget http://mirrors.kernel.org/ubuntu/pool/main/o/openssl/openssl_1.1.1f-1ubuntu2.19_amd64.deb && \ - dpkg -i openssl_1.1.1f-1ubuntu2.19_amd64.deb && \ - rm openssl_1.1.1f-1ubuntu2.19_amd64.deb + +RUN curl -L https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-linux-amd64.tar.gz -o mergestat.tar.gz && \ + tar zxf mergestat.tar.gz -C /usr/local/bin/ && \ + rm mergestat.tar.gz # The best developer experience for load testing -ENV K6_VERSION=v0.44.0 +ENV K6_VERSION=v0.47.0 RUN curl -L https://github.com/grafana/k6/releases/download/${K6_VERSION}/k6-${K6_VERSION}-linux-amd64.tar.gz -o k6.tar.gz && \ tar xvf k6.tar.gz && \ mv k6-${K6_VERSION}-linux-amd64/k6 /usr/local/bin/k6 && \ rm k6.tar.gz # Benthos is a high performance and resilient stream processor -RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.15.0 +RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.22.0 # Commandline tool for running SQL queries against JSON, CSV, Excel, Parquet, and more RUN curl -L https://github.com/multiprocessio/dsq/releases/download/v0.23.0/dsq-linux-x64-v0.23.0.zip -o dsq.zip && \ - unzip dsq.zip && \ + unzip -q dsq.zip && \ mv dsq /usr/local/bin/dsq && \ rm dsq.zip # Install alexellis/arkade as root RUN curl -sLS https://get.arkade.dev | sh -# Install Azure CLI (need to install as root) -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash RUN mkdir /opt/database && groupadd --gid 1000 canary && \ useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \ @@ -95,14 +100,15 @@ ENV PATH="${PATH}:/var/lib/canary/bin/" # Install AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \ + unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \ rm awscliv2.zip # Install GCP CLI -RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \ - tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \ +ENV GCLOUD_VERSION=441.0.0 +RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ + tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \ - rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz + rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz COPY --from=builder /app/.bin/canary-checker /app diff --git a/build/minimal/Dockerfile b/build/minimal/Dockerfile index fa6628729..277e332b5 100644 --- a/build/minimal/Dockerfile +++ b/build/minimal/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder +FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder WORKDIR /app ARG NAME @@ -12,22 +12,30 @@ RUN go mod download COPY ./ ./ RUN make build -FROM ubuntu +FROM ubuntu:jammy-20231004@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f WORKDIR /app +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \ - rm -Rf /var/lib/apt/lists/* && \ - rm -Rf /usr/share/doc && rm -Rf /usr/share/man && \ + apt-get install -y curl unzip ca-certificates jq tzdata wget gnupg2 bzip2 apt-transport-https lsb-release --no-install-recommends && \ apt-get clean -COPY --from=builder /app/.bin/canary-checker /app +RUN mkdir -p /etc/apt/keyrings && \ + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \ + gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \ + chmod go+r /etc/apt/keyrings/microsoft.gpg && \ + echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \ + cat /etc/apt/sources.list.d/azure-cli.list && \ + apt-get update && \ + apt-get install -y azure-cli && \ + apt-get clean + +RUN apt-get update && apt-get upgrade -y && \ + rm -Rf /var/lib/apt/lists/* && \ + apt-get clean # Install alexellis/arkade as root RUN curl -sLS https://get.arkade.dev | sh -# Install Azure CLI (need to install as root) -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash - RUN mkdir /opt/database && groupadd --gid 1000 canary && \ useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \ chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app @@ -41,15 +49,17 @@ ENV PATH="${PATH}:/var/lib/canary/bin/" # Install AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \ + unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \ rm awscliv2.zip # Install GCP CLI -RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \ - tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \ +ENV GCLOUD_VERSION=441.0.0 +RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ + tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \ - rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz + rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz -RUN /app/canary-checker go-offline +COPY --from=builder /app/.bin/canary-checker /app +RUN /app/canary-checker go-offline ENTRYPOINT ["/app/canary-checker"] diff --git a/checks/github.go b/checks/github.go index 983df0834..9518d7a05 100644 --- a/checks/github.go +++ b/checks/github.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" osExec "os/exec" - "strings" "github.com/flanksource/canary-checker/api/context" "github.com/flanksource/canary-checker/api/external" @@ -49,27 +48,23 @@ func (c *GitHubChecker) Check(ctx *context.Context, extConfig external.Check) pk } } - askGitCmd := fmt.Sprintf("GITHUB_TOKEN=%v askgit \"%v\" --format json", githubToken, check.Query) + askGitCmd := fmt.Sprintf("mergestat \"%v\" --format json", check.Query) + if ctx.IsTrace() { + ctx.Tracef("Executing askgit command: %v", askGitCmd) + } cmd := osExec.Command("bash", "-c", askGitCmd) + cmd.Env = append(cmd.Env, "GITHUB_TOKEN="+githubToken) output, err := cmd.CombinedOutput() if err != nil { return results.Failf("error executing askgit command. output=%q: %v", output, err) } - rows := string(output) - var rowResults = make([]map[string]string, 0) - for _, row := range strings.Split(rows, "\n") { - if row == "" { - continue - } - var rowResult map[string]string - err := json.Unmarshal([]byte(row), &rowResult) - if err != nil { - return results.Failf("error parsing askgit result: %v", err) - } - - rowResults = append(rowResults, rowResult) + var rowResults = make([]map[string]any, 0) + err = json.Unmarshal(output, &rowResults) + if err != nil { + return results.Failf("error parsing mergestat result: %v", err) } + result.AddDetails(rowResults) return results } diff --git a/fixtures/git/_setup.sh b/fixtures/git/_setup.sh index 6ba43a463..4232e2b2e 100755 --- a/fixtures/git/_setup.sh +++ b/fixtures/git/_setup.sh @@ -2,21 +2,15 @@ set -e -# Install askgit -curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz -tar xf askgit.tar.gz -sudo mv askgit /usr/bin/askgit -sudo chmod +x /usr/bin/askgit -rm askgit.tar.gz +if ! which mergestat > /dev/null; then + if $(uname -a | grep -q Darwin); then + curl -L https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-macos-amd64.tar.gz -o mergestat.tar.gz + sudo tar xf mergestat.tar.gz -C /usr/local/bin/ -wget -O libssl.deb http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb -sudo dpkg -i libssl.deb - -#verification -which askgit -if ! askgit --help > /dev/null; then - printf "`askgit --help` failed. Check the binary?" - exit 1; + else + curl -L https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-linux-amd64.tar.gz -o mergestat.tar.gz + sudo tar xf mergestat.tar.gz -C /usr/local/bin/ + fi fi # creating a GITHUB_TOKEN Secret diff --git a/fixtures/git/git_check_pass.yaml b/fixtures/git/git_check_pass.yaml index 13c4eb1b2..dfda47e4f 100644 --- a/fixtures/git/git_check_pass.yaml +++ b/fixtures/git/git_check_pass.yaml @@ -2,10 +2,12 @@ apiVersion: canaries.flanksource.com/v1 kind: Canary metadata: name: github-pass + annotations: + trace: "true" spec: interval: 30 github: - - query: "SELECT * FROM github_repo_checks('flanksource/duty') where branch='main'" + - query: "SELECT * FROM commits('https://github.com/flanksource/commons')" name: github-check test: expr: size(results) > 0 diff --git a/fixtures/git/git_test_expression_pass.yaml b/fixtures/git/git_test_expression_pass.yaml index 5d2517094..a1c6dc37e 100644 --- a/fixtures/git/git_test_expression_pass.yaml +++ b/fixtures/git/git_test_expression_pass.yaml @@ -13,4 +13,4 @@ spec: valueFrom: secretKeyRef: name: github-token - key: GITHUB_TOKEN \ No newline at end of file + key: GITHUB_TOKEN