Skip to content

Commit

Permalink
Merge branch 'develop' into 4499-version-doi
Browse files Browse the repository at this point in the history
  • Loading branch information
poikilotherm committed Jun 13, 2023
2 parents 288e987 + fd190a3 commit e5dd1a6
Show file tree
Hide file tree
Showing 232 changed files with 18,285 additions and 1,978 deletions.
96 changes: 96 additions & 0 deletions .github/workflows/container_app_pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
---
name: Preview Application Container Image

on:
# We only run the push commands if we are asked to by an issue comment with the correct command.
# This workflow is always taken from the default branch and runs in repo context with access to secrets.
repository_dispatch:
types: [ push-image-command ]

env:
IMAGE_TAG: unstable
BASE_IMAGE_TAG: unstable
PLATFORMS: "linux/amd64,linux/arm64"

jobs:
deploy:
name: "Package & Push"
runs-on: ubuntu-latest
# Only run in upstream repo - avoid unnecessary runs in forks
if: ${{ github.repository_owner == 'IQSS' }}
steps:
# Checkout the pull request code as when merged
- uses: actions/checkout@v3
with:
ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
- uses: actions/setup-java@v3
with:
java-version: "11"
distribution: 'adopt'
- uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2

# Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
- name: Login to Github Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}

- name: Set up QEMU for multi-arch builds
uses: docker/setup-qemu-action@v2

# Get the image tag from either the command or default to branch name (Not used for now)
#- name: Get the target tag name
# id: vars
# run: |
# tag=${{ github.event.client_payload.slash_command.args.named.tag }}
# if [[ -z "$tag" ]]; then tag=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-'); fi
# echo "IMAGE_TAG=$tag" >> $GITHUB_ENV

# Set image tag to branch name of the PR
- name: Set image tag to branch name
run: |
echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
# Necessary to split as otherwise the submodules are not available (deploy skips install)
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
run: >
mvn -B -f modules/dataverse-parent
-P ct -pl edu.harvard.iq:dataverse -am
install
- name: Deploy multi-arch application and configbaker container image
run: >
mvn
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
-Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
-Pct deploy
- uses: marocchino/sticky-pull-request-comment@v2
with:
header: registry-push
hide_and_recreate: true
hide_classify: "OUTDATED"
number: ${{ github.event.client_payload.pull_request.number }}
message: |
:package: Pushed preview images as
```
ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
```
```
ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
```
:ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
# Leave a note when things have gone sideways
- uses: peter-evans/create-or-update-comment@v3
if: ${{ failure() }}
with:
issue-number: ${{ github.event.client_payload.pull_request.number }}
body: >
:package: Could not push preview images :disappointed:.
See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.
109 changes: 87 additions & 22 deletions .github/workflows/container_app_push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,19 @@ on:
- master
paths:
- 'src/main/docker/**'
- 'modules/container-configbaker/**'
- '.github/workflows/container_app_push.yml'

env:
IMAGE_TAG: unstable
BASE_IMAGE_TAG: unstable
REGISTRY: "" # Empty means default to Docker Hub
PLATFORMS: "linux/amd64,linux/arm64"
MASTER_BRANCH_TAG: alpha

jobs:
build:
name: Build & deploy
name: "Build & Test"
runs-on: ubuntu-latest
permissions:
contents: read
Expand All @@ -38,30 +40,75 @@ jobs:
uses: actions/setup-java@v3
with:
java-version: "11"
distribution: 'adopt'
- name: Cache Maven packages
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
distribution: temurin
cache: maven

- name: Build app container image with local architecture
run: mvn -Pct package
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
run: >
mvn -B -f modules/dataverse-parent
-P ct -pl edu.harvard.iq:dataverse -am
install
# Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
# TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")

# Run this when triggered via push or schedule as reused workflow from base / maven unit tests
- if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' }}
name: Push description to DockerHub
uses: peter-evans/dockerhub-description@v3
hub-description:
needs: build
name: Push image descriptions to Docker Hub
# Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
# Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: gdcc/dataverse
short-description: "Dataverse Application Container Image providing the executable"
readme-filepath: ./src/main/docker/README.md
- uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: gdcc/configbaker
short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
readme-filepath: ./modules/container-configbaker/README.md

# Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
# We check for them here and subsequent jobs can rely on this to decide if they shall run.
check-secrets:
needs: build
name: Check for Secrets Availability
runs-on: ubuntu-latest
outputs:
available: ${{ steps.secret-check.outputs.available }}
steps:
- id: secret-check
# perform secret check & put boolean result as an output
shell: bash
run: |
if [ "${{ secrets.DOCKERHUB_TOKEN }}" != '' ]; then
echo "available=true" >> $GITHUB_OUTPUT;
else
echo "available=false" >> $GITHUB_OUTPUT;
fi
deploy:
needs: check-secrets
name: "Package & Publish"
runs-on: ubuntu-latest
# Only run this job if we have access to secrets. This is true for events like push/schedule which run in
# context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
if: needs.check-secrets.outputs.available == 'true'
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: "11"
distribution: temurin

# Depending on context, we push to different targets. Login accordingly.
- if: ${{ github.event_name != 'pull_request' }}
name: Log in to Docker Hub registry
uses: docker/login-action@v2
Expand All @@ -82,21 +129,39 @@ jobs:
- name: Re-set image tag based on branch (if master)
if: ${{ github.ref_name == 'master' }}
run: |
echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
echo "BASE_IMAGE_TAG=alpha" >> $GITHUB_ENV
echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
- name: Re-set image tag and container registry when on PR
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
- name: Deploy multi-arch application container image
run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
# Necessary to split as otherwise the submodules are not available (deploy skips install)
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
run: >
mvn -B -f modules/dataverse-parent
-P ct -pl edu.harvard.iq:dataverse -am
install
- name: Deploy multi-arch application and configbaker container image
run: >
mvn
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
-P ct deploy
- uses: marocchino/sticky-pull-request-comment@v2
if: ${{ github.event_name == 'pull_request' }}
with:
header: app-registry-push
header: registry-push
hide_and_recreate: true
hide_classify: "OUTDATED"
message: |
Pushed preview application image as [`ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}`](https://github.com/orgs/gdcc/packages/container/package/dataverse).
Use it by referencing it with its full name as printed above.
:package: Pushed preview images as
```
ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
```
```
ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
```
:ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
38 changes: 27 additions & 11 deletions .github/workflows/maven_unit_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@ on:
- "**.java"
- "pom.xml"
- "modules/**/pom.xml"
- "!modules/container-base/**"
- "!modules/dataverse-spi/**"
pull_request:
paths:
- "**.java"
- "pom.xml"
- "modules/**/pom.xml"
- "!modules/container-base/**"
- "!modules/dataverse-spi/**"

jobs:
unittest:
Expand All @@ -33,25 +37,37 @@ jobs:
continue-on-error: ${{ matrix.experimental }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v2
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.jdk }}
distribution: 'adopt'
- name: Cache Maven packages
uses: actions/cache@v2
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
distribution: temurin
cache: maven

# The reason why we use "install" here is that we want the submodules to be available in the next step.
# Also, we can cache them this way for jobs triggered by this one.
- name: Build with Maven
run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test
run: >
mvn -B -f modules/dataverse-parent
-Dtarget.java.version=${{ matrix.jdk }}
-DcompilerArgument=-Xlint:unchecked -P all-unit-tests
-pl edu.harvard.iq:dataverse -am
install
- name: Maven Code Coverage
env:
CI_NAME: github
COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
# The coverage commit is sometimes flaky. Don't bail out just because this optional step failed.
continue-on-error: true
run: >
mvn -B
-DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
jacoco:report coveralls:report
# We don't want to cache the WAR file, so delete it
- run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
push-app-img:
name: Publish App Image
permissions:
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/pr_comment_commands.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: PR Comment Commands
on:
issue_comment:
types: [created]
jobs:
dispatch:
# Avoid being triggered by forks in upstream
if: ${{ github.repository_owner == 'IQSS' }}
runs-on: ubuntu-latest
steps:
- name: Dispatch
uses: peter-evans/slash-command-dispatch@v3
with:
# This token belongs to @dataversebot and has sufficient scope.
token: ${{ secrets.GHCR_TOKEN }}
commands: |
push-image
repository: IQSS/dataverse
# Commenter must have at least write permission to repo to trigger dispatch
permission: write
35 changes: 29 additions & 6 deletions .github/workflows/shellcheck.yml
Original file line number Diff line number Diff line change
@@ -1,24 +1,47 @@
name: "Shellcheck"
on:
push:
branches:
- develop
paths:
- conf/solr/**
- modules/container-base/**
- conf/solr/**/.sh
- modules/container-base/**/*.sh
- modules/container-configbaker/**/*.sh
pull_request:
branches:
- develop
paths:
- conf/solr/**
- modules/container-base/**
- conf/solr/**/*.sh
- modules/container-base/**/*.sh
- modules/container-configbaker/**/*.sh
jobs:
shellcheck:
name: Shellcheck
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: shellcheck
uses: reviewdog/action-shellcheck@v1
with:
github_token: ${{ secrets.github_token }}
reporter: github-pr-review # Change reporter.
fail_on_error: true
# Container base image uses dumb-init shebang, so nail to using bash
shellcheck_flags: "--shell=bash --external-sources"
shellcheck_flags: "--shell=bash --external-sources"
# Exclude old scripts
exclude: |
*/.git/*
conf/docker-aio/*
doc/*
downloads/*
scripts/database/*
scripts/globalid/*
scripts/icons/*
scripts/installer/*
scripts/issues/*
scripts/r/*
scripts/tests/*
scripts/vagrant/*
tests/*
Loading

0 comments on commit e5dd1a6

Please sign in to comment.