Feature/k8s support

.github/workflows/compliance-test.yaml

@@ -1,11 +1,11 @@
 # Workflow for running the TES compliance suite against Funnel
-
+#
 # This includes the following steps:
 #  1. Build Funnel and store the resulting binary artifact
 #  2. Install tes-compliance-suite and run against every version of TES simultaneously
 #  3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
 #     the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
-
+#
 # Optionally debug via SSH
 # Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
 #
@@ -29,10 +29,9 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    container: quay.io/ohsu-comp-bio/slurm
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
 
@@ -43,7 +42,7 @@ jobs:
       run: make build
 
     - name: Store funnel
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: funnelBin
         path: funnel
@@ -54,12 +53,9 @@ jobs:
       matrix:
         version: [1.0.0, 1.1.0]
         db: ["boltdb", "mongodb"]
-        compute: ["local", "slurm"]
+        compute: ["local"]
     needs: build
     runs-on: ubuntu-latest
-    container:
-      image: quay.io/ohsu-comp-bio/slurm
-      options: --hostname slurmctl --cap-add sys_admin
     steps:
       # Required to access the 'tests/mongo.config.yml' file
       # Perhaps uploading it as an artifact would be more efficient?
@@ -76,60 +72,17 @@ jobs:
           if [ ${{ matrix.db }} = "mongodb" ]; then
             make start-mongodb
             cat `pwd`/tests/mongo.config.yml >> config.yml
-            # Required for Funnel to connect MongoDB
-            echo "172.17.0.1 localhost" >> /etc/hosts
-          elif [ ${{ matrix.compute }} = "slurm" ]; then
-            cat `pwd`/tests/slurm.config.yml >> config.yml
-            cp config.yml /opt/funnel_config.yml
-            # Start Slurm
-            /usr/local/bin/docker-entrypoint.sh
           fi
           chmod +x funnel
-          FLAGS="--config `pwd`/config.yml"
-          ./funnel server run $FLAGS &> funnel.logs &
+          ./funnel server run --config `pwd`/config.yml &> funnel.logs &
 
       - name: Run OpenAPI Test Runner
         run: |
-          # Clone the 'upstream' OpenAPI Test Runner when PR #65 is merged
-          # https://github.com/elixir-cloud-aai/openapi-test-runner/pull/65
-          # git clone https://github.com/elixir-cloud-aai/openapi-test-runner
-          git clone https://github.com/ohsu-comp-bio/openapi-test-runner -b fix/create-and-filter-task
-
+          git clone https://github.com/elixir-cloud-aai/openapi-test-runner
           cd openapi-test-runner
           python3 -m venv venv
           source venv/bin/activate
           pip install -r requirements.txt
           python setup.py install  
           openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
 
-      - name: Install TES compliance suite
-        run: |
-          git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1
-          cd tes-compliance-suite
-          /root/.pyenv/shims/python3 -m venv venv
-          source venv/bin/activate
-          pip install -r requirements.txt
-          python setup.py install
-          mkdir reports
-
-      - name: Test compliance (Report Publishing)
-        run: |
-          cd tes-compliance-suite
-          source venv/bin/activate
-          tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/"
-
-  start-report-deployment:
-    needs: compliance
-    runs-on: ubuntu-latest
-    steps:
-      # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event
-      - name: Start report generation
-        uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_AUTHOR: "Funnel"
-          PAYLOAD_REVISION: "3"
-        with:
-          repository: ohsu-comp-bio/funnel-compliance
-          event: start-report
-          github_pat: ${{ secrets.ACTIONS_TOKEN }}
-

.github/workflows/nextflow.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
 
@@ -19,7 +19,7 @@ jobs:
       run: make build
 
     - name: Store Funnel
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: funnelBin
         path: funnel
@@ -42,14 +42,12 @@ jobs:
 
     - name: Install Nextflow
       run: |
-        cd ..
         git clone https://github.com/nextflow-io/nextflow
         cd nextflow
         make compile
 
     - name: Install nf-canary and GA4GH-TES plugin
       run: |
-        cd ..
         git clone https://github.com/seqeralabs/nf-canary
         cd nf-canary
         cat <<EOF >> nextflow.config
@@ -59,8 +57,7 @@ jobs:
           process.executor = 'tes'
           tes.endpoint = 'http://localhost:8000'
         EOF
-  
+
     - name: Run nf-canary tests
       run: |
-        cd ../nf-canary
-        ../nextflow/launch.sh run main.nf
+        ./nextflow/nextflow run nf-canary/main.nf

.github/workflows/s3-test.yaml

@@ -5,15 +5,32 @@ name: S3 Integration Test
 
 on:
   push:
-    branches:
-      - main
 
 jobs:
-  s3Test-integration:
+  build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.21
 
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: make build
+
+    - name: Store funnel
+      uses: actions/upload-artifact@v4
+      with:
+        name: funnelBin
+        path: funnel
+
+  s3Test:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
       - name: Setup minio
         run: |
           docker run -d -p 9000:9000 --name minio \
@@ -23,26 +40,29 @@ jobs:
                      -v /tmp/config:/root/.minio \
                      minio/minio server /data
 
-          cat <<EOF > funnel.config.yml
-          LocalStorage:
-            Disabled: true
-          AmazonS3:
-            Disabled: true
-          GoogleStorage:
-            Disabled: true
-          HTTPStorage:
-            Disabled: true
-          FTPStorage:
-            Disabled: true
-          GenericS3:
-            - Disabled: false
-              Endpoint: "localhost:9000"
-              Key: "minioadmin"
-              Secret: "minioadmin"
-          EOF
+      - uses: actions/download-artifact@v3
+        with:
+          name: funnelBin
 
-          wget https://github.com/ohsu-comp-bio/funnel/releases/download/untagged-217841a99d14ccfe289c/funnel-darwin-arm64-0.11.0.tar.gz
-          tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz
+      - name: Start Funnel server
+        run: |
+          cat <<EOF > config.yml
+            LocalStorage:
+              Disabled: true
+            AmazonS3:
+              Disabled: true
+            GoogleStorage:
+              Disabled: true
+            HTTPStorage:
+              Disabled: true
+            FTPStorage:
+              Disabled: true
+            GenericS3:
+              - Disabled: false
+                Endpoint: "localhost:9000"
+                Key: "minioadmin"
+                Secret: "minioadmin"
+          EOF
           chmod +x funnel
-          ./funnel server run --config funnel.config.yml &
+          ./funnel server run --config `pwd`/config.yml &> funnel.logs &
           ./funnel task run examples/s3-test.yml

.github/workflows/tests.yaml

@@ -4,26 +4,26 @@ on:
   push:
 
 jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.21
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          version: latest
-          # TODO: Re-enable all linters
-          args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign -D typecheck
+  # Temporarily disabling linting
+  # lint:
+  #   name: lint
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/setup-go@v3
+  #       with:
+  #         go-version: 1.21
+  #     - uses: actions/checkout@v3
+  #     - name: golangci-lint
+  #       uses: golangci/golangci-lint-action@v3
+  #       with:
+  #         version: latest
+  #         args: --timeout 3m --verbose
 
   build:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
 
@@ -34,7 +34,7 @@ jobs:
       run: make build
 
     - name: Store funnel
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: funnelBin
         path: funnel
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
     - name: Check out code
@@ -57,7 +57,7 @@ jobs:
     needs: build
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
 
@@ -81,7 +81,7 @@ jobs:
     needs: build
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
     - name: Check out code
@@ -101,7 +101,7 @@ jobs:
     needs: build
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
     - name: Check out code
@@ -122,7 +122,7 @@ jobs:
     needs: build
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
         go-version: 1.21
     - name: Check out code

.goreleaser.yaml

@@ -28,10 +28,10 @@ archives:
     name_template: "{{.ProjectName}}-{{.Os}}-{{.Arch}}-{{.Version}}"
 
 brews:
-  - tap:
+  - repository:
       owner: ohsu-comp-bio
       name: homebrew-formula
-    folder: Formula
+    directory: Formula
     skip_upload: true
     description: "distributed task execution toolkit"
     homepage: "https://ohsu-comp-bio.github.io/funnel/"

Dockerfile

@@ -1,5 +1,5 @@
 # build stage
-FROM golang:1.21-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 RUN apk add make git bash build-base
 ENV GOPATH=/go
 ENV PATH="/go/bin:${PATH}"

Dockerfile.dind

@@ -1,13 +1,13 @@
 # build stage
-FROM golang:1.20-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 RUN apk add make git bash build-base
 ENV GOPATH=/go
 ENV PATH="/go/bin:${PATH}"
 ADD ./ /go/src/github.com/ohsu-comp-bio/funnel
 RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build
 
 # final stage
-FROM docker:stable-dind
+FROM docker:dind
 WORKDIR /opt/funnel
 VOLUME /opt/funnel/funnel-work-dir
 EXPOSE 8000 9090

Dockerfile.dind-rootless

@@ -1,13 +1,13 @@
 # build stage
-FROM golang:1.20-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 RUN apk add make git bash build-base
 ENV GOPATH=/go
 ENV PATH="/go/bin:${PATH}"
 ADD ./ /go/src/github.com/ohsu-comp-bio/funnel
 RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build
 
 # final stage
-FROM docker:stable-dind-rootless
+FROM docker:dind-rootless
 WORKDIR /opt/funnel
 VOLUME /opt/funnel/funnel-work-dir
 EXPOSE 8000 9090

Makefile

@@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git
 export GIT_BRANCH = $(git_branch)
 export GIT_UPSTREAM = $(git_upstream)
 
-export FUNNEL_VERSION=0.11.0
+export FUNNEL_VERSION=0.11.1-rc.5
 
 # LAST_PR_NUMBER is used by the release notes builder to generate notes
 # based on pull requests (PR) up until the last release.