Skip to content

Commit

Permalink
Merge branch 'feature/k8s-support' into feature/container-engines
Browse files Browse the repository at this point in the history
  • Loading branch information
lbeckman314 committed Sep 30, 2024
2 parents 234e934 + b257941 commit 934646a
Show file tree
Hide file tree
Showing 10 changed files with 518 additions and 171 deletions.
88 changes: 88 additions & 0 deletions .github/workflows/compliance-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# Workflow for running the TES compliance suite against Funnel
#
# This includes the following steps:
# 1. Build Funnel and store the resulting binary artifact
# 2. Install tes-compliance-suite and run against every version of TES simultaneously
# 3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
# the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
#
# Optionally debug via SSH
# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
#
# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and
# output a ssh address associated with the Github action worker. Helpful for debugging build steps and
# and intermediary files/artifacts.
#
# - name: "Debug: Package dependancies for tmate (CentOS)"
# run: |
# yum install -y xz
# ln -s /bin/true /bin/apt-get
#
# - name: Setup tmate session
# uses: mxschmitt/action-tmate@v3

name: Compliance Test

on:
push:

jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version: 1.21

- name: Check out code
uses: actions/checkout@v2

- name: Build
run: make build

- name: Store funnel
uses: actions/upload-artifact@v4
with:
name: funnelBin
path: funnel

compliance:
strategy:
fail-fast: false
matrix:
version: [1.0.0, 1.1.0]
db: ["boltdb", "mongodb"]
compute: ["local"]
needs: build
runs-on: ubuntu-latest
steps:
# Required to access the 'tests/mongo.config.yml' file
# Perhaps uploading it as an artifact would be more efficient?
- name: Check out code
uses: actions/checkout@v2

- uses: actions/download-artifact@v3
with:
name: funnelBin

- name: Start Funnel server
run: |
touch config.yml
if [ ${{ matrix.db }} = "mongodb" ]; then
make start-mongodb
cat `pwd`/tests/mongo.config.yml >> config.yml
fi
chmod +x funnel
./funnel server run --config `pwd`/config.yml &> funnel.logs &
- name: Run OpenAPI Test Runner
run: |
git clone https://github.com/elixir-cloud-aai/openapi-test-runner
cd openapi-test-runner
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
python setup.py install
openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
19 changes: 18 additions & 1 deletion .github/workflows/nextflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,25 @@ on:

jobs:
build:
uses: ./.github/workflows/build.yml
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version: 1.21

- name: Check out code
uses: actions/checkout@v2

- name: Build Funnel (if cache does not exist)
run: make build

- name: Store Funnel
uses: actions/upload-artifact@v4
with:
name: funnelBin
path: funnel

nextflow:
needs: build
runs-on: ubuntu-latest
Expand Down
68 changes: 68 additions & 0 deletions .github/workflows/s3-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Credit: rhnvrm
# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/

name: S3 Integration Test

on:
push:

jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version: 1.21

- name: Check out code
uses: actions/checkout@v2

- name: Build
run: make build

- name: Store funnel
uses: actions/upload-artifact@v4
with:
name: funnelBin
path: funnel

s3Test:
needs: build
runs-on: ubuntu-latest
steps:
- name: Setup minio
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ROOT_USER=minioadmin" \
-e "MINIO_ROOT_PASSWORD=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
- uses: actions/download-artifact@v3
with:
name: funnelBin

- name: Start Funnel server
run: |
cat <<EOF > config.yml
LocalStorage:
Disabled: true
AmazonS3:
Disabled: true
GoogleStorage:
Disabled: true
HTTPStorage:
Disabled: true
FTPStorage:
Disabled: true
GenericS3:
- Disabled: false
Endpoint: "localhost:9000"
Key: "minioadmin"
Secret: "minioadmin"
EOF
chmod +x funnel
./funnel server run --config `pwd`/config.yml &> funnel.logs &
./funnel task run examples/s3-test.yml
62 changes: 32 additions & 30 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,38 +5,40 @@ on:
pull_request:

jobs:
lint:
name: lint
# Temporarily disabling linting
# lint:
# name: lint
# runs-on: ubuntu-latest
# steps:
# - uses: actions/setup-go@v3
# with:
# go-version: 1.21
# - uses: actions/checkout@v3
# - name: golangci-lint
# uses: golangci/golangci-lint-action@v3
# with:
# version: latest
# args: --timeout 3m --verbose

build:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.21

- uses: actions/checkout@v3

- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: latest
# Matches the "primary" golangci-lint command in the Makefile
args: |
--timeout 3m --disable-all --enable=govet --enable=gofmt --enable=goimports --enable=misspell \
--skip-dirs "vendor" \
--skip-dirs "webdash" \
--skip-dirs "cmd/webdash" \
--skip-dirs "funnel-work-dir" \
-e '.*bundle.go' -e ".*pb.go" -e ".*pb.gw.go" \
./...
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: latest
# Matches the "termdash" golangci-lint command in the Makefile
args: |
--timeout 3m --disable-all --enable=vet --enable=gofmt --enable=goimports --enable=misspell \
./cmd/termdash/...
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version: 1.21

- name: Check out code
uses: actions/checkout@v2

- name: Build
run: make build

- name: Store funnel
uses: actions/upload-artifact@v4
with:
name: funnelBin
path: funnel

unitTest:
runs-on: ubuntu-latest
Expand Down
23 changes: 14 additions & 9 deletions deployments/kubernetes/funnel-deployment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,20 @@ spec:
serviceAccountName: funnel-sa
containers:
- name: funnel
image: ohsucompbio/funnel:latest
imagePullPolicy: Never
args:
- "server"
- "run"
- "--config"
- "/etc/config/funnel-server-config.yml"
resources:
requests:
image: quay.io/ohsu-comp-bio/funnel:latest
imagePullPolicy: IfNotPresent
command:
- 'funnel'
- 'server'
- 'run'
- '--config'
- '/etc/config/funnel-server-config.yml'
resources:
requests:
cpu: 500m
memory: 1G
ephemeral-storage: 25G
limits:
cpu: 2000m
memory: 4G
ephemeral-storage: 25G
Expand Down
72 changes: 54 additions & 18 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,24 @@ require (
)

require (
cloud.google.com/go v0.115.1 // indirect
cloud.google.com/go/auth v0.9.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/iam v1.1.13 // indirect
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.6.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.8 // indirect
cloud.google.com/go/kms v1.17.1 // indirect
cloud.google.com/go/longrunning v0.5.7 // indirect
cloud.google.com/go/storage v1.41.0 // indirect
code.gitea.io/sdk/gitea v0.18.0 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/AlekSi/pointer v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.29 // indirect
Expand Down Expand Up @@ -212,11 +225,19 @@ require (
github.com/google/go-github/v62 v62.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/ko v0.15.4 // indirect
github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/google/wire v0.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/goreleaser/chglog v0.6.1 // indirect
github.com/goreleaser/fileglob v1.3.0 // indirect
github.com/goreleaser/goreleaser v1.26.2 // indirect
github.com/goreleaser/nfpm/v2 v2.37.1 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
Expand Down Expand Up @@ -250,7 +271,10 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect
Expand Down Expand Up @@ -346,16 +370,28 @@ require (
gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/term v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
go.opentelemetry.io/otel v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/automaxprocs v1.5.3 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
gocloud.dev v0.37.0 // indirect
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
Expand Down
Loading

0 comments on commit 934646a

Please sign in to comment.