Skip to content

Commit

Permalink
Merge branch 'master' into feat/upstream-jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
moshloop authored Oct 18, 2023
2 parents 5b0474f + 6faffed commit 9b8fca1
Show file tree
Hide file tree
Showing 18 changed files with 148 additions and 134 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@ permissions: read-all
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
target:
- docker-full
- docker-minimal
steps:
- name: Checkout code
uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
- name: Build Container
run: make docker
run: make ${{matrix.target}}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
- { name: k8s, on: ubuntu-latest }
- { name: datasources, on: ubuntu-latest }
- { name: opensearch, on: ubuntu-latest }
- { name: elasticsearch, on: ubuntu-latest }
# - { name: elasticsearch, on: ubuntu-latest }
- { name: git, on: ubuntu-latest }
# - restic
runs-on: ${{ matrix.suite.on }}
Expand Down
9 changes: 6 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,15 +82,18 @@ generate: .bin/controller-gen
.bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./api/..."

# Build the docker image
docker:
docker build . -f build/full/Dockerfile -t ${IMG_F}
docker: docker-minimal docker-full

docker-full:
docker build . -f build/full/Dockerfile -t ${IMG}

docker-minimal:
docker build . -f build/minimal/Dockerfile -t ${IMG}

# Build the docker image
docker-dev: linux
docker build ./ -f build/dev/Dockerfile -t ${IMG}


docker-push-%:
docker build . -f build/full/Dockerfile -t ${IMG_F}
docker build . -f build/minimal/Dockerfile -t ${IMG}
Expand Down
2 changes: 1 addition & 1 deletion api/v1/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ type HTTPCheck struct {
ResponseCodes []int `yaml:"responseCodes,omitempty" json:"responseCodes,omitempty"`
// Exact response content expected to be returned by the endpoint.
ResponseContent string `yaml:"responseContent,omitempty" json:"responseContent,omitempty"`
// Path and value to of expect JSON response by the endpoint
// Deprecated, use expr and jsonpath function
ResponseJSONContent *JSONCheck `yaml:"responseJSONContent,omitempty" json:"responseJSONContent,omitempty"`
// Maximum number of days until the SSL Certificate expires.
MaxSSLExpiry int `yaml:"maxSSLExpiry,omitempty" json:"maxSSLExpiry,omitempty"`
Expand Down
74 changes: 40 additions & 34 deletions build/full/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder
FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder
WORKDIR /app

ARG NAME
Expand All @@ -12,27 +12,37 @@ RUN go mod download
COPY ./ ./
RUN make build

FROM eclipse-temurin:11.0.18_10-jdk-focal@sha256:509043cc38d37a5bd44720b471c38bef40fb34de67c03baaa67a5a9d8cda52a0
FROM eclipse-temurin:11.0.20.1_1-jdk-jammy@sha256:1584fd589b45a67b6f56b0c702776ca3d5640d1001f7848f5bcd19cb10545eaa
WORKDIR /app
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \
apt-get install -y curl unzip ca-certificates jq tzdata wget gnupg2 bzip2 apt-transport-https lsb-release python3 python3-pip --no-install-recommends && \
rm -Rf /var/lib/apt/lists/* && \
rm -Rf /usr/share/doc && rm -Rf /usr/share/man && \
apt-get clean

RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
apt-get update && apt-get install -y \
google-chrome-stable \
fontconfig \
fonts-ipafont-gothic \
fonts-wqy-zenhei \
fonts-thai-tlwg \
fonts-kacst \
fonts-symbola \
fonts-noto \
fonts-freefont-ttf \
--no-install-recommends
RUN pip3 install pip pyyaml lxml requests robotframework \
robotframework \
robotframework-jsonlibrary \
robotframework-jsonschemalibrary \
robotframework-requests \
robotframework-restlibrary \
robotframework-seleniumlibrary \
robotframework-excellib \
robotframework-crypto \
robotframework-databaselibrary \
psycopg2-binary \
PyMySQL

RUN mkdir -p /etc/apt/keyrings && \
curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \
gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \
chmod go+r /etc/apt/keyrings/microsoft.gpg && \
echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \
cat /etc/apt/sources.list.d/azure-cli.list && \
apt-get update && \
apt-get install -y azure-cli && \
apt-get clean

RUN apt-get update && apt-get upgrade -y && \
rm -Rf /var/lib/apt/lists/* && \
Expand All @@ -45,42 +55,37 @@ RUN curl -L https://github.com/restic/restic/releases/download/v${RESTIC_VERSION
mv /app/restic /usr/local/bin/ && \
rm -rf /app/restic.bz2

ENV JMETER_VERSION=5.5
ENV JMETER_VERSION=5.6.2
RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip -o apache-jmeter-${JMETER_VERSION}.zip && \
unzip apache-jmeter-${JMETER_VERSION}.zip -d /opt && \
unzip -q apache-jmeter-${JMETER_VERSION}.zip -d /opt && \
rm apache-jmeter-${JMETER_VERSION}.zip

ENV PATH /opt/apache-jmeter-${JMETER_VERSION}/bin/:$PATH

RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz && \
tar xf askgit.tar.gz && \
mv askgit /usr/local/bin/askgit && \
rm askgit.tar.gz && \
wget http://mirrors.kernel.org/ubuntu/pool/main/o/openssl/openssl_1.1.1f-1ubuntu2.19_amd64.deb && \
dpkg -i openssl_1.1.1f-1ubuntu2.19_amd64.deb && \
rm openssl_1.1.1f-1ubuntu2.19_amd64.deb

RUN curl -L https://github.com/mergestat/mergestat-lite/releases/download/v0.6.1/mergestat-linux-amd64.tar.gz -o mergestat.tar.gz && \
tar zxf mergestat.tar.gz -C /usr/local/bin/ && \
rm mergestat.tar.gz

# The best developer experience for load testing
ENV K6_VERSION=v0.44.0
ENV K6_VERSION=v0.47.0
RUN curl -L https://github.com/grafana/k6/releases/download/${K6_VERSION}/k6-${K6_VERSION}-linux-amd64.tar.gz -o k6.tar.gz && \
tar xvf k6.tar.gz && \
mv k6-${K6_VERSION}-linux-amd64/k6 /usr/local/bin/k6 && \
rm k6.tar.gz

# Benthos is a high performance and resilient stream processor
RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.15.0
RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.22.0

# Commandline tool for running SQL queries against JSON, CSV, Excel, Parquet, and more
RUN curl -L https://github.com/multiprocessio/dsq/releases/download/v0.23.0/dsq-linux-x64-v0.23.0.zip -o dsq.zip && \
unzip dsq.zip && \
unzip -q dsq.zip && \
mv dsq /usr/local/bin/dsq && \
rm dsq.zip

# Install alexellis/arkade as root
RUN curl -sLS https://get.arkade.dev | sh

# Install Azure CLI (need to install as root)
RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash

RUN mkdir /opt/database && groupadd --gid 1000 canary && \
useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \
Expand All @@ -95,14 +100,15 @@ ENV PATH="${PATH}:/var/lib/canary/bin/"

# Install AWS CLI
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
rm awscliv2.zip

# Install GCP CLI
RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
ENV GCLOUD_VERSION=441.0.0
RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \
rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz
rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz

COPY --from=builder /app/.bin/canary-checker /app

Expand Down
38 changes: 24 additions & 14 deletions build/minimal/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.20@sha256:690e4135bf2a4571a572bfd5ddfa806b1cb9c3dea0446ebadaf32bc2ea09d4f9 AS builder
FROM golang:1.20-bookworm@sha256:077ff85b374b23916b4b41835e242e5a3ddad9fc537ea7e980f230431747d245 AS builder
WORKDIR /app

ARG NAME
Expand All @@ -12,22 +12,30 @@ RUN go mod download
COPY ./ ./
RUN make build

FROM ubuntu
FROM ubuntu:jammy-20231004@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f
WORKDIR /app
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y curl unzip ca-certificates jq wget gnupg2 bzip2 --no-install-recommends && \
rm -Rf /var/lib/apt/lists/* && \
rm -Rf /usr/share/doc && rm -Rf /usr/share/man && \
apt-get install -y curl unzip ca-certificates jq tzdata wget gnupg2 bzip2 apt-transport-https lsb-release --no-install-recommends && \
apt-get clean

COPY --from=builder /app/.bin/canary-checker /app
RUN mkdir -p /etc/apt/keyrings && \
curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \
gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \
chmod go+r /etc/apt/keyrings/microsoft.gpg && \
echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \
cat /etc/apt/sources.list.d/azure-cli.list && \
apt-get update && \
apt-get install -y azure-cli && \
apt-get clean

RUN apt-get update && apt-get upgrade -y && \
rm -Rf /var/lib/apt/lists/* && \
apt-get clean

# Install alexellis/arkade as root
RUN curl -sLS https://get.arkade.dev | sh

# Install Azure CLI (need to install as root)
RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash

RUN mkdir /opt/database && groupadd --gid 1000 canary && \
useradd canary --uid 1000 -g canary -m -d /var/lib/canary && \
chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app
Expand All @@ -41,15 +49,17 @@ ENV PATH="${PATH}:/var/lib/canary/bin/"

# Install AWS CLI
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
unzip -q awscliv2.zip && ./aws/install -i ${HOME}/aws -b ${HOME}/bin/ && \
rm awscliv2.zip

# Install GCP CLI
RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
tar -xf google-cloud-cli-441.0.0-linux-x86_64.tar.gz && \
ENV GCLOUD_VERSION=441.0.0
RUN curl -sL -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
tar -xf google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
ln -sf /app/google-cloud-sdk/bin/gcloud ${HOME}/bin/gcloud && \
rm google-cloud-cli-441.0.0-linux-x86_64.tar.gz
rm google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz

RUN /app/canary-checker go-offline
COPY --from=builder /app/.bin/canary-checker /app

RUN /app/canary-checker go-offline
ENTRYPOINT ["/app/canary-checker"]
25 changes: 10 additions & 15 deletions checks/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
osExec "os/exec"
"strings"

"github.com/flanksource/canary-checker/api/context"
"github.com/flanksource/canary-checker/api/external"
Expand Down Expand Up @@ -49,27 +48,23 @@ func (c *GitHubChecker) Check(ctx *context.Context, extConfig external.Check) pk
}
}

askGitCmd := fmt.Sprintf("GITHUB_TOKEN=%v askgit \"%v\" --format json", githubToken, check.Query)
askGitCmd := fmt.Sprintf("mergestat \"%v\" --format json", check.Query)
if ctx.IsTrace() {
ctx.Tracef("Executing askgit command: %v", askGitCmd)
}
cmd := osExec.Command("bash", "-c", askGitCmd)
cmd.Env = append(cmd.Env, "GITHUB_TOKEN="+githubToken)
output, err := cmd.CombinedOutput()
if err != nil {
return results.Failf("error executing askgit command. output=%q: %v", output, err)
}

rows := string(output)
var rowResults = make([]map[string]string, 0)
for _, row := range strings.Split(rows, "\n") {
if row == "" {
continue
}
var rowResult map[string]string
err := json.Unmarshal([]byte(row), &rowResult)
if err != nil {
return results.Failf("error parsing askgit result: %v", err)
}

rowResults = append(rowResults, rowResult)
var rowResults = make([]map[string]any, 0)
err = json.Unmarshal(output, &rowResults)
if err != nil {
return results.Failf("error parsing mergestat result: %v", err)
}

result.AddDetails(rowResults)
return results
}
Loading

0 comments on commit 9b8fca1

Please sign in to comment.