From 8de4ba0b5c965b4a1852abb2556f340b317b2c27 Mon Sep 17 00:00:00 2001 From: Stacky McStackface Date: Mon, 7 Feb 2022 15:02:19 +0000 Subject: [PATCH] Generated commit to update templated files based on rev a89f13e in stackabletech/operator-templating repo. Original commit message: Make copying of config-spec conditional (#62) Some operators (looking at you, Regorule) do not have config-spec files, which caused the generate manifest step to fail after PR #61 which changed the syntax for copying these files. This adds a check if any config-spec files are present and skips copying them if not. This requires that operators do not have .dummy files in place, as was the case for regorule (fixed in https://github.com/stackabletech/regorule-operator/pull/195 ) --- .flake8 | 2 + .github/dependabot.yml | 30 -- .github/workflows/build.yml | 308 ++++++++++++++++++ .github/workflows/pr_generate_manifests.yml | 39 --- .github/workflows/publish_main_artifacts.yml | 59 ---- .github/workflows/publish_pr_artifacts.yml | 62 ---- .../workflows/publish_release_artifacts.yml | 56 ---- .github/workflows/reviewdog.yaml | 70 ++++ .github/workflows/rust.yml | 126 ------- Makefile | 13 +- bors.toml | 7 +- deploy/DO_NOT_EDIT.md | 10 +- .../configs/config-spec/properties.yaml | 120 ------- deploy/manifests/configmap.yaml | 5 +- deploy/manifests/crds.yaml | 2 + deploy/manifests/deployment.yaml | 12 +- deploy/manifests/roles.yaml | 1 + deploy/manifests/serviceaccount.yaml | 6 + python/cargo-version.py | 177 ---------- python/cargo_version.py | 71 ++-- renovate.json | 31 ++ scripts/generate-manifests.sh | 18 +- 22 files changed, 508 insertions(+), 717 deletions(-) create mode 100644 .flake8 delete mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/pr_generate_manifests.yml delete mode 100644 .github/workflows/publish_main_artifacts.yml delete mode 100644 .github/workflows/publish_pr_artifacts.yml delete mode 100644 .github/workflows/publish_release_artifacts.yml create mode 100644 .github/workflows/reviewdog.yaml delete mode 100644 .github/workflows/rust.yml delete mode 100644 deploy/helm/hive-operator/configs/config-spec/properties.yaml delete mode 100755 python/cargo-version.py create mode 100644 renovate.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..ea6646f3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E111,E501,E114 diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 50eaadb2..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,30 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= ---- -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - labels: - - "type/dependencies" - reviewers: - - "stackabletech/developers" - - - package-ecosystem: "cargo" - directory: "/" - schedule: - interval: "weekly" - labels: - - "type/dependencies" - reviewers: - - "stackabletech/rust-developers" - ignore: - # We never want to be notified about a kube-rs update. - # It often contains breaking changes so it has to be updated manually anyway - # and it needs to be updated together with kube-runtime, kube-derive etc. - - dependency-name: "kube*" - - dependency-name: "k8s-openapi" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..7643882c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,308 @@ +--- +name: Stackable Build Pipeline + +on: + push: + branches: + - main + - staging + - trying + - "renovate/**" + tags: + - "*" + pull_request: + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: '0' + CARGO_PROFILE_DEV_DEBUG: '0' + RUSTFLAGS: "-D warnings" + RUSTDOCFLAGS: "-D warnings" + RUST_LOG: "info" + PRODUCT_NAME: hive + DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev + TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test + STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable + +jobs: + # Identify unused dependencies + run_udeps: + name: Run Cargo Udeps + runs-on: ubuntu-latest + env: + RUSTC_BOOTSTRAP: 1 + steps: + - uses: actions/checkout@v2.4.0 + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v1.3.0 + with: + key: udeps + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-udeps --locked + - uses: actions-rs/cargo@v1 + with: + command: udeps + + # This job evaluates the github environment to determine why this action is running and selects the appropriate + # target repository for published Helm charts based on this. + # + # The following scenarios are identified: + # - pull request: + # condition: github.event_name == "pull_request" + # repository: test + # + # - release (aka a tag was created): + # condition: github.event_name == 'create' & github.ref.startswith('refs/tags/') + # repository: stable + # + # - merge of pr to main branch: + # condition: github.event_name == 'push' & github.ref == 'refs/heads/main' + # repository: dev + # + # Any other scenarios will cause the publish step to be skipped, most commonly this is expected to happen for the + # branches that bors uses internally (staging, trying) for which the checks need to run, but we do not want artifacts + # to be published. + select_repo: + name: Select target repository based on action trigger + runs-on: ubuntu-latest + outputs: + repository: ${{ steps.selectrepo.outputs.repo }} + steps: + - id: selectrepo + env: + TRIGGER: ${{ github.event_name }} + GITHUB_REF: ${{ github.ref }} + run: | + if [[ $TRIGGER == "pull_request" ]]; then + echo "exporting test as target repo: ${{ env.TEST_REPO_HELM_URL }}" + echo "::set-output name=repo::${{ env.TEST_REPO_HELM_URL }}" + elif [[ $TRIGGER == "push" && $GITHUB_REF == "refs/heads/main" ]]; then + echo "exporting dev as target repo: ${{ env.DEV_REPO_HELM_URL }}" + echo "::set-output name=repo::${{ env.DEV_REPO_HELM_URL }}" + elif [[ ( $TRIGGER == "create" || $TRIGGER == "push" ) && $GITHUB_REF == refs/tags/* ]]; then + echo "exporting stable as target repo: ${{ env.STABLE_REPO_HELM_URL }}" + echo "::set-output name=repo::${{ env.STABLE_REPO_HELM_URL }}" + else + echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF" + echo "::set-output name=repo::skip" + fi + + run_cargodeny: + name: Run Cargo Deny + runs-on: ubuntu-latest + strategy: + matrix: + checks: + - advisories + - bans licenses sources + + # Prevent sudden announcement of a new advisory from failing ci: + continue-on-error: ${{ matrix.checks == 'advisories' }} + + steps: + - uses: actions/checkout@v2.4.0 + - uses: EmbarkStudios/cargo-deny-action@v1.2.10 + with: + command: check ${{ matrix.checks }} + + run_rustfmt: + name: Run Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + components: rustfmt + override: true + - uses: actions-rs/cargo@v1.0.3 + with: + command: fmt + args: --all -- --check + + run_clippy: + name: Run Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + components: clippy + override: true + - uses: Swatinem/rust-cache@v1.3.0 + with: + key: clippy + - name: Run clippy action to produce annotations + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions-rs/clippy-check@v1.0.7 + if: env.GITHUB_TOKEN != null + with: + args: --all-targets -- -D warnings + token: ${{ secrets.GITHUB_TOKEN }} + - name: Run clippy manually without annotations + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: env.GITHUB_TOKEN == null + run: cargo clippy --all-targets -- -D warnings + + run_rustdoc: + name: Run RustDoc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + components: rustfmt + override: true + - uses: Swatinem/rust-cache@v1.3.0 + with: + key: doc + - uses: actions-rs/cargo@v1.0.3 + with: + command: doc + args: --document-private-items + + run_tests: + name: Run Cargo Tests + needs: + - run_cargodeny + - run_clippy + - run_rustfmt + - run_rustdoc + - run_udeps + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v1.3.0 + with: + key: test + - uses: actions-rs/cargo@v1.0.3 + with: + command: test + + # This job cleans up the CRDs, Helm charts and Kustomize manifests, followed by rebuilding them + # It then runs a `git diff` and fails the entire workflow, if any difference is encountered. + # + # Since CRD files are generated during the 'cargo build' process we need to run this once after + # removing the CRD files to ensure that the checked in versions match what the code expects. + # + # The reason for this step is, that developers are expected to check in up-to-date versions of charts + # and manifests, as we'd otherwise have to build these in CI and commit them back to the PR, which + # creates all kinds of problems. + # Therefor this failsafe simply aborts anything that has not had charts and manifests rebuilt before pushing. + check_charts: + name: Check if committed Helm & Kustomize Charts were up to date + needs: + - run_cargodeny + - run_clippy + - run_rustfmt + - run_rustdoc + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Helm + uses: azure/setup-helm@v2.0 + with: + version: v3.6.2 + - name: Set up cargo + uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + override: true + - name: Set up rust-cache + uses: Swatinem/rust-cache@v1.3.0 + with: + key: charts + - name: Regenerate charts + run: make regenerate-charts + - name: Check if committed charts were up to date + run: git diff --exit-code + - name: Git Diff showed uncommitted changes + if: ${{ failure() }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('Committed charts were not up to date, please regenerate and re-commit!') + + test_charts: + name: Run Chart Tests + needs: + - check_charts + - run_tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: placeholder + run: echo Tests will go here + + package_and_publish: + name: Package Charts, Build Docker Image and publish them + needs: + - test_charts + - select_repo + runs-on: ubuntu-latest + env: + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + REPO: ${{ needs.select_repo.outputs.repository }} + if: needs.select_repo.outputs.repository != 'skip' + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + if: ${{ github.event_name == 'pull_request' }} + - uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + components: rustfmt + override: true + - name: Install requirements for version tool + if: ${{ github.event_name == 'pull_request' }} + run: pip install -r python/requirements.txt + + # This step checks if the current run was triggered by a push to a pr (or a pr being created). + # If this is the case it changes the version of this project in all Cargo.toml files to include the suffix + # "-pr" so that the published artifacts can be linked to this PR. + - name: Update version if PR + if: ${{ github.event_name == 'pull_request' }} + run: python/cargo_version.py -m pr${{ github.event.pull_request.number }} + + # Recreate charts with changed version if needed + - name: Clean charts + if: ${{ github.event_name == 'pull_request' }} + run: make chart-clean clean-manifests compile-chart generate-manifests + + # Package and publish charts + - name: Package Chart + run: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${{ env.PRODUCT_NAME }}-operator + - name: Build Docker image + if: env.NEXUS_PASSWORD != null + run: make docker + - name: Publish Chart + if: env.NEXUS_PASSWORD != null + run: >- + /usr/bin/curl + --fail + -u 'github:${{ secrets.NEXUS_PASSWORD }}' + --upload-file "./$(find target/helm/ -name '*.tgz')" + "${{ env.REPO }}/" diff --git a/.github/workflows/pr_generate_manifests.yml b/.github/workflows/pr_generate_manifests.yml deleted file mode 100644 index c480eedf..00000000 --- a/.github/workflows/pr_generate_manifests.yml +++ /dev/null @@ -1,39 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= -name: Update Manifest files - -on: - pull_request: - -jobs: - manifests: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - token: ${{ secrets.STACKY_MC_STACKFACE_TOKEN }} - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: v3.6.2 - - name: update manifests - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: make generate-manifests - - name: Add & Commit - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - uses: EndBug/add-and-commit@v7 - with: - default_author: user_info - author_name: Stacky McStackface - author_email: stackable-bot@users.noreply.github.com - pathspec_error_handling: exitImmediately - pull: NO-PULL - add: 'deploy' - message: 'Github Actions: Generated k8s manifest files' diff --git a/.github/workflows/publish_main_artifacts.yml b/.github/workflows/publish_main_artifacts.yml deleted file mode 100644 index a9bfd787..00000000 --- a/.github/workflows/publish_main_artifacts.yml +++ /dev/null @@ -1,59 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= ---- -name: Publish nightly artifacts from main branch - -on: - push: - branches: - - main - schedule: - - cron: '30 4 * * *' - workflow_dispatch: - -env: - PRODUCT_NAME: hive - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: '0' - CARGO_PROFILE_DEV_DEBUG: '0' - RUSTFLAGS: "-D warnings" - REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev - -jobs: - helm: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: v3.6.2 - - - name: Build Docker image - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: make docker - - - name: Compile chart - run: make compile-chart - - - name: Package Chart - run: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${{ env.PRODUCT_NAME }}-operator - - - name: Publish Chart - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: >- - /usr/bin/curl - --fail - -u 'github:${{ secrets.NEXUS_PASSWORD }}' - --upload-file "./$(find target/helm/ -name '*.tgz')" - "${{ env.REPO_HELM_URL }}/" diff --git a/.github/workflows/publish_pr_artifacts.yml b/.github/workflows/publish_pr_artifacts.yml deleted file mode 100644 index 0e3dc91c..00000000 --- a/.github/workflows/publish_pr_artifacts.yml +++ /dev/null @@ -1,62 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= ---- -name: Publish pull-request artifacts - -on: - pull_request: - -env: - PRODUCT_NAME: hive - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: '0' - CARGO_PROFILE_DEV_DEBUG: '0' - RUSTFLAGS: "-D warnings" - REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test - -jobs: - helm: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: v3.6.2 - - - name: Set up Python and update cargo version. - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - run: pip install -r ./python/requirements.txt - - run: python ./python/cargo_version.py -m pr${{ github.event.number }} - - - name: Build Docker image - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: make docker - - - name: Compile chart - run: make compile-chart - - - name: Package Chart - run: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${{ env.PRODUCT_NAME }}-operator - - - name: Publish Chart - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: >- - /usr/bin/curl - --fail - -u 'github:${{ secrets.NEXUS_PASSWORD }}' - --upload-file "./$(find target/helm/ -name '*.tgz')" - "${{ env.REPO_HELM_URL }}/" diff --git a/.github/workflows/publish_release_artifacts.yml b/.github/workflows/publish_release_artifacts.yml deleted file mode 100644 index 0990606c..00000000 --- a/.github/workflows/publish_release_artifacts.yml +++ /dev/null @@ -1,56 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= ---- -name: Publish release artifacts - -on: - push: - tags: - - "*" - -env: - PRODUCT_NAME: hive - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: '0' - CARGO_PROFILE_DEV_DEBUG: '0' - RUSTFLAGS: "-D warnings" - REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable - -jobs: - helm: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: v3.6.2 - - - name: Build Docker image - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: make docker-release - - - name: Compile chart - run: make compile-chart - - - name: Package Chart - run: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${{ env.PRODUCT_NAME }}-operator - - - name: Publish Chart - env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - if: env.NEXUS_PASSWORD != null - run: >- - /usr/bin/curl - --fail - -u 'github:${{ secrets.NEXUS_PASSWORD }}' - --upload-file "./$(find target/helm/ -name '*.tgz')" - "${{ env.REPO_HELM_URL }}/" diff --git a/.github/workflows/reviewdog.yaml b/.github/workflows/reviewdog.yaml new file mode 100644 index 00000000..398684d4 --- /dev/null +++ b/.github/workflows/reviewdog.yaml @@ -0,0 +1,70 @@ +--- +name: reviewdog +on: + pull_request + +permissions: + contents: read + checks: write + pull-requests: write + issues: write + +jobs: + actionlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: reviewdog/action-actionlint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + detect-secrets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: reviewdog/action-detect-secrets@master + with: + github_token: ${{ secrets.github_token }} + + flake8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + - uses: reviewdog/action-flake8@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + hadolint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: reviewdog/action-hadolint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + markdownlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: reviewdog/action-markdownlint@v0.1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + shellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: reviewdog/action-shellcheck@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + yamllint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: reviewdog/action-yamllint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index e7333134..00000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,126 +0,0 @@ -# ============= -# This file is automatically generated from the templates in stackabletech/operator-templating -# DON'T MANUALLY EDIT THIS FILE -# ============= ---- -name: Rust checks - -on: - push: - branches: - - main - - staging - - trying - pull_request: - -env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: '0' - CARGO_PROFILE_DEV_DEBUG: '0' - RUSTFLAGS: "-D warnings" - RUSTDOCFLAGS: "-D warnings" - RUST_LOG: "info" - -jobs: - - test: - name: Run tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - - uses: actions-rs/toolchain@v1.0.7 - with: - profile: minimal - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v1.3.0 - with: - key: test - - uses: actions-rs/cargo@v1.0.3 - with: - command: test - - rustfmt: - name: Run rustfmt - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - - uses: actions-rs/toolchain@v1.0.7 - with: - profile: minimal - toolchain: stable - components: rustfmt - override: true - - uses: actions-rs/cargo@v1.0.3 - with: - command: fmt - args: --all -- --check - - doc: - name: Run rustdoc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - - uses: actions-rs/toolchain@v1.0.7 - with: - profile: minimal - toolchain: stable - components: rustfmt - override: true - - uses: Swatinem/rust-cache@v1.3.0 - with: - key: doc - - uses: actions-rs/cargo@v1.0.3 - with: - command: doc - args: --document-private-items - - clippy: - name: Run clippy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - - uses: actions-rs/toolchain@v1.0.7 - with: - profile: minimal - toolchain: stable - components: clippy - override: true - - uses: Swatinem/rust-cache@v1.3.0 - with: - key: clippy - # We need this due to: https://github.com/actions-rs/clippy-check/issues/2 - - name: Check workflow permissions - id: check_permissions - uses: scherermichael-oss/action-has-permission@1.0.6 - with: - required-permission: write - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Run clippy action to produce annotations - uses: actions-rs/clippy-check@v1.0.7 - if: steps.check_permissions.outputs.has-permission - with: - args: --all-targets -- -D warnings - token: ${{ secrets.GITHUB_TOKEN }} - - name: Run clippy manually without annotations - if: ${{ !steps.check_permissions.outputs.has-permission }} - run: cargo clippy --all-targets -- -D warnings - - cargo-deny: - name: Run cargo deny - runs-on: ubuntu-latest - strategy: - matrix: - checks: - - advisories - - bans licenses sources - - # Prevent sudden announcement of a new advisory from failing ci: - continue-on-error: ${{ matrix.checks == 'advisories' }} - - steps: - - uses: actions/checkout@v2.4.0 - - uses: EmbarkStudios/cargo-deny-action@v1.2.6 - with: - command: check ${{ matrix.checks }} diff --git a/Makefile b/Makefile index 68a78560..ace15b07 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,10 @@ version: yq eval -i '.version = ${VERSION} | .appVersion = ${VERSION}' deploy/helm/hive-operator/Chart.yaml config: - cp -r deploy/config-spec deploy/helm/hive-operator/configs + if [ -d "deploy/config-spec/" ]; then\ + mkdir -p deploy/helm/hive-operator/configs;\ + cp -r deploy/config-spec/* deploy/helm/hive-operator/configs;\ + fi crds: mkdir -p deploy/helm/hive-operator/crds @@ -55,3 +58,11 @@ clean-manifests: generate-manifests: clean-manifests compile-chart ./scripts/generate-manifests.sh + +clean-crds: + rm -rf deploy/crd/* + +generate-crds: + cargo build + +regenerate-charts: clean-crds chart-clean clean-manifests generate-crds compile-chart generate-manifests diff --git a/bors.toml b/bors.toml index 4f305075..420893f1 100644 --- a/bors.toml +++ b/bors.toml @@ -1,12 +1,9 @@ status = [ - 'Run tests', - 'Run rustfmt', - 'Run rustdoc', - 'Run clippy', - 'Run cargo deny (bans licenses sources)' + 'Package Charts, Build Docker Image and publish them' ] delete_merged_branches = true use_squash_merge = true pr_status = [ 'license/cla' ] timeout_sec = 7200 cut_body_after = "" +required_approvals = 1 diff --git a/deploy/DO_NOT_EDIT.md b/deploy/DO_NOT_EDIT.md index da37bf18..8ff382e8 100644 --- a/deploy/DO_NOT_EDIT.md +++ b/deploy/DO_NOT_EDIT.md @@ -1,4 +1,10 @@ These Helm charts and manifests are automatically generated. -Please do not edit anything in this directory manually. +Please do not edit anything except for files explicitly mentioned below in this +directory manually. -The details are in-motion but check this repository for a few details: https://github.com/stackabletech/operator-templating +The following files are ok to edit: + +- helm/hive-operator/templates/roles.yaml + +The details are in-motion but check this repository for a few details: + diff --git a/deploy/helm/hive-operator/configs/config-spec/properties.yaml b/deploy/helm/hive-operator/configs/config-spec/properties.yaml deleted file mode 100644 index 542a9cc9..00000000 --- a/deploy/helm/hive-operator/configs/config-spec/properties.yaml +++ /dev/null @@ -1,120 +0,0 @@ -version: 0.1.0 -spec: - units: - - unit: &unitUri - name: "uri" - regex: "^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?" - examples: - - "ldap://[2001:db8::7]/c=GB?objectClass?one" - comment: "Specified in https://tools.ietf.org/html/rfc3986#appendix-B" - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitClassName - name: "classname" - regex: "([a-zA-Z_$][a-zA-Z\\d_$]*\\.)*[a-zA-Z_$][a-zA-Z\\d_$]*" - examples: - - "org.apache.derby.jdbc.EmbeddedDriver" - -properties: - - property: - propertyNames: - - name: "javax.jdo.option.ConnectionURL" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "string" - # unit: *unitUri - roles: - - name: "metastore" - required: true - asOfVersion: "0.0.0" - description: "JDBC connect string for a JDBC metastore. - To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL. - For example, jdbc:postgresql://myhost/db?ssl=true for postgres database." - - property: - propertyNames: - - name: "javax.jdo.option.ConnectionDriverName" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "string" - unit: *unitClassName - roles: - - name: "metastore" - required: true - asOfVersion: "0.0.0" - - property: - propertyNames: - - name: "javax.jdo.option.ConnectionUserName" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "string" - roles: - - name: "metastore" - required: true - asOfVersion: "0.0.0" - - property: - propertyNames: - - name: "javax.jdo.option.ConnectionPassword" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "string" - roles: - - name: "metastore" - required: true - asOfVersion: "0.0.0" - - property: - propertyNames: - - name: "hive.metastore.port" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "integer" - min: "1024" - max: "65535" - defaultValues: - - value: "9083" - roles: - - name: "metastore" - required: true - asOfVersion: "0.0.0" - - property: - propertyNames: - - name: "hive.metastore.metrics.enabled" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "bool" - defaultValues: - - value: "false" - roles: - - name: "metastore" - required: false - asOfVersion: "0.0.0" - - property: - propertyNames: - - name: "hive.metastore.warehouse.dir" - kind: - type: "file" - file: "hive-site.xml" - datatype: - type: "string" - unit: *unitUri - defaultValues: - - value: "/user/hive/warehouse" - roles: - - name: "metastore" - required: false - description: "URI of the default location for native tables." - asOfVersion: "0.0.0" diff --git a/deploy/manifests/configmap.yaml b/deploy/manifests/configmap.yaml index ded7b93f..1701c46e 100644 --- a/deploy/manifests/configmap.yaml +++ b/deploy/manifests/configmap.yaml @@ -1,4 +1,5 @@ --- +# Source: hive-operator/templates/configmap.yaml apiVersion: v1 data: properties.yaml: | @@ -21,7 +22,7 @@ data: regex: "([a-zA-Z_$][a-zA-Z\\d_$]*\\.)*[a-zA-Z_$][a-zA-Z\\d_$]*" examples: - "org.apache.derby.jdbc.EmbeddedDriver" - + properties: - property: propertyNames: @@ -126,6 +127,8 @@ kind: ConfigMap metadata: name: hive-operator-configmap labels: + helm.sh/chart: hive-operator-0.7.0-nightly app.kubernetes.io/name: hive-operator app.kubernetes.io/instance: hive-operator app.kubernetes.io/version: "0.7.0-nightly" + app.kubernetes.io/managed-by: Helm diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 9c3edb9c..fbdde688 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -1,4 +1,5 @@ --- +# Source: hive-operator/crds/crds.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -240,3 +241,4 @@ spec: storage: true subresources: status: {} + diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index 83075187..84758c25 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -1,12 +1,15 @@ --- +# Source: hive-operator/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: hive-operator-deployment labels: + helm.sh/chart: hive-operator-0.7.0-nightly app.kubernetes.io/name: hive-operator app.kubernetes.io/instance: hive-operator app.kubernetes.io/version: "0.7.0-nightly" + app.kubernetes.io/managed-by: Helm spec: replicas: 1 strategy: @@ -22,13 +25,16 @@ spec: app.kubernetes.io/instance: hive-operator spec: serviceAccountName: hive-operator-serviceaccount - securityContext: {} + securityContext: + {} containers: - name: hive-operator - securityContext: {} + securityContext: + {} image: "docker.stackable.tech/stackable/hive-operator:0.7.0-nightly" imagePullPolicy: IfNotPresent - resources: {} + resources: + {} volumeMounts: - mountPath: /etc/stackable/hive-operator/config-spec name: config-spec diff --git a/deploy/manifests/roles.yaml b/deploy/manifests/roles.yaml index af10b14f..055443eb 100644 --- a/deploy/manifests/roles.yaml +++ b/deploy/manifests/roles.yaml @@ -1,4 +1,5 @@ --- +# Source: hive-operator/templates/roles.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/deploy/manifests/serviceaccount.yaml b/deploy/manifests/serviceaccount.yaml index 1af80aec..bb0ea953 100644 --- a/deploy/manifests/serviceaccount.yaml +++ b/deploy/manifests/serviceaccount.yaml @@ -1,22 +1,28 @@ --- +# Source: hive-operator/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: hive-operator-serviceaccount labels: + helm.sh/chart: hive-operator-0.7.0-nightly app.kubernetes.io/name: hive-operator app.kubernetes.io/instance: hive-operator app.kubernetes.io/version: "0.7.0-nightly" + app.kubernetes.io/managed-by: Helm --- +# Source: hive-operator/templates/serviceaccount.yaml apiVersion: rbac.authorization.k8s.io/v1 # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. kind: ClusterRoleBinding metadata: name: hive-operator-clusterrolebinding labels: + helm.sh/chart: hive-operator-0.7.0-nightly app.kubernetes.io/name: hive-operator app.kubernetes.io/instance: hive-operator app.kubernetes.io/version: "0.7.0-nightly" + app.kubernetes.io/managed-by: Helm subjects: - kind: ServiceAccount name: hive-operator-serviceaccount diff --git a/python/cargo-version.py b/python/cargo-version.py deleted file mode 100755 index 474240dd..00000000 --- a/python/cargo-version.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 -# -# Utility for viewing and managing versions of cargo workspaces and crates. -# For workspaces, it assumes that all crate members use a single shared version. -# -# usage: cargo-version.py [-h] [-p PROJECT] [-r] [-n {major,minor,patch}] [-s SET] [-o] -# -# Change versions of cargo projects. -# -# optional arguments: -# -h, --help show this help message and exit -# -p PROJECT, --project PROJECT -# Project folder -# -r, --release Version -# -n {major,minor,patch}, --next {major,minor,patch} -# Version -# -s SET, --set SET Version -# -o, --show Version -# - -import toml -import semver -import argparse - -class Crate: - def __init__(self, path, name, version, dependencies): - self.path = path - self.name = name - self.version = version - self.dependencies = dependencies - - def with_dependencies(self, names): - deps = {k:v for k,v in self.dependencies.items() if k in names} - return Crate(self.path, self.name, self.version, deps) - - @classmethod - def finalize(cls, version): - return str(semver.VersionInfo.parse(version).finalize_version()) - - @classmethod - def bump_level(cls, version, level): - v = semver.VersionInfo.parse(version) - if level == 'major': - return str(v.bump_major()) - elif level == 'minor': - return str(v.bump_minor()) - elif level == 'patch': - return str(v.bump_patch()) - else: - return str(v.bump_prerelease('nightly'))[:-2] ### remove the .1 suffix that semver always adds to the prererelease. - - @classmethod - def prerelease(cls, version, prerelease): - v = semver.VersionInfo.parse(version) - return str(semver.VersionInfo(v.major, v.minor, v.patch, prerelease)) - - def finalize_version(self): - return Crate(self.path, self.name, Crate.finalize(self.version), self.dependencies.copy()) - - def bump_version(self, level): - return Crate(self.path, self.name, Crate.bump_level(self.version, level), self.dependencies.copy()) - - def set_version(self, version): - return Crate(self.path, self.name, version, self.dependencies.copy()) - - def set_prerelease(self, prerelease): - return Crate(self.path, self.name, Crate.prerelease(self.version, prerelease), self.dependencies.copy()) - - def next_version(self): - return Crate(self.path, self.name, str(semver.VersionInfo.parse(self.version).next_version('patch')), self.dependencies.copy()) - - def show_version(self): - return self.version - - def save(self, previous): - contents = [] - cargo_file = f"{self.path}/Cargo.toml" - with open(cargo_file, 'r') as r: - for line in r.readlines(): - if line.startswith("version"): - line = line.replace(previous.version, self.version) - else: - for dname, dversion in self.dependencies.items(): - if line.startswith(dname): - line = line.replace(previous.dependencies[dname], dversion) - contents.append(line) - - with open(cargo_file, 'w') as w: - w.write(''.join(contents)) - - def __str__(self): - return f'Crate({self.path}, {self.name}, {self.version}, {self.dependencies})' - -class Workspace: - def __init__(self, crates): - names = set([c.name for c in crates]) - self.crates = {c.name: c.with_dependencies(names) for c in crates} - - def finalize_version(self): - crates = {c.name: c.finalize_version() for c in self.crates.values()} - return Workspace(Workspace.update_dependencies(crates).values()) - - def bump_version(self, level): - crates = {c.name: c.bump_version(level) for c in self.crates.values()} - return Workspace(Workspace.update_dependencies(crates).values()) - - def set_version(self, version): - crates = {c.name: c.set_version(version) for c in self.crates.values()} - return Workspace(Workspace.update_dependencies(crates).values()) - - def set_prerelease(self, prerelease): - crates = {c.name: c.set_prerelease(prerelease) for c in self.crates.values()} - return Workspace(Workspace.update_dependencies(crates).values()) - - def next_version(self): - crates = {c.name: c.next_version() for c in self.crates.values()} - return Workspace(Workspace.update_dependencies(crates).values()) - - def show_version(self): - for c in self.crates.values(): - return c.show_version() - return "0.0.0" - - @classmethod - def update_dependencies(cls, crate_dict): - for crate in crate_dict.values(): - for dep in crate.dependencies.keys(): - crate.dependencies[dep] = crate_dict[dep].version - return crate_dict - - def __str__(self): - return f'Workspace({[str(c) for c in self.crates.values()]})' - - def save(self, previous): - for cn in self.crates.keys(): - self.crates[cn].save(previous.crates[cn]) - -def load(root): - r = toml.load(f"{root}/Cargo.toml") - if "workspace" in r: - return Workspace([load(f"{root}/{path}") for path in r["workspace"]["members"]]) - else: - return Crate(path=root, name=r["package"]["name"], version=r["package"]["version"], dependencies={dn: r["dependencies"][dn]["version"] for dn in r["dependencies"] if "version" in r["dependencies"][dn]}) - -def parse_args(): - parser = argparse.ArgumentParser(description="Change versions of cargo projects.") - parser.add_argument("-p", "--project", help="Project folder", default=".") - parser.add_argument("-r", "--release", help="Version", action="store_true") - parser.add_argument("-n", "--next", help="Version", choices=['major', 'minor', 'patch']) - parser.add_argument("-s", "--set", help="Version" ) - parser.add_argument("-o", "--show", help="Version", action="store_true") - parser.add_argument("-m", "--prerelease", help="Set pre-prelease string." ) - return parser.parse_args() - -if __name__ == "__main__": - args = parse_args() - - old = load(args.project.rstrip('/')) - - if args.release: - new = old.finalize_version() - new.save(old) - elif args.next: - new = old.bump_version(args.next).bump_version("prerelease") - new.save(old) - elif args.set: - # sanity check - semver.VersionInfo.parse(args.set) - new = old.set_version(args.set) - new.save(old) - elif args.prerelease: - new = old.set_prerelease(args.prerelease) - new.save(old) - elif args.show: - print(old.show_version()) - - diff --git a/python/cargo_version.py b/python/cargo_version.py index c783ce58..aa67cb01 100755 --- a/python/cargo_version.py +++ b/python/cargo_version.py @@ -3,7 +3,9 @@ # Utility for viewing and managing versions of cargo workspaces and crates. # For workspaces, it assumes that all crate members use a single shared version. # -# usage: cargo_version.py [-h] [-p PROJECT] [-r] [-n {major,minor,patch}] [-s SET] [-o] +# usage: +# cargo_version.py [-h] [-p PROJECT] [-r] [-n {major,minor,patch}] +# [-s SET] [-o] [-m PRERELEASE] # # Change versions of cargo projects. # @@ -16,6 +18,9 @@ # Version # -s SET, --set SET Version # -o, --show Version +# -m PRERELEASE, --prerelease PRERELEASE +# Set pre-prelease string. +# import argparse import semver @@ -39,35 +44,42 @@ def finalize(cls, version): @classmethod def bump_level(cls, version, level): - v = semver.VersionInfo.parse(version) + parsed_version = semver.VersionInfo.parse(version) if level == 'major': - return str(v.bump_major()) + return str(parsed_version.bump_major()) if level == 'minor': - return str(v.bump_minor()) + return str(parsed_version.bump_minor()) if level == 'patch': - return str(v.bump_patch()) - else: - return str(v.bump_prerelease('nightly'))[:-2] ### remove the .1 suffix that semver always adds to the prererelease. + return str(parsed_version.bump_patch()) + + ### remove the .1 suffix that semver always adds to the prererelease before returning + return str(parsed_version.bump_prerelease('nightly'))[:-2] @classmethod def prerelease(cls, version, prerelease): - v = semver.VersionInfo.parse(version) - return str(semver.VersionInfo(v.major, v.minor, v.patch, prerelease)) + parsed_version = semver.VersionInfo.parse(version) + return str(semver.VersionInfo(parsed_version.major, parsed_version.minor, + parsed_version.patch, prerelease)) def finalize_version(self): - return Crate(self.path, self.name, Crate.finalize(self.version), self.dependencies.copy()) + return Crate(self.path, self.name, Crate.finalize(self.version), + self.dependencies.copy()) def bump_version(self, level): - return Crate(self.path, self.name, Crate.bump_level(self.version, level), self.dependencies.copy()) + return Crate(self.path, self.name, Crate.bump_level(self.version, level), + self.dependencies.copy()) def set_version(self, version): return Crate(self.path, self.name, version, self.dependencies.copy()) def set_prerelease(self, prerelease): - return Crate(self.path, self.name, Crate.prerelease(self.version, prerelease), self.dependencies.copy()) + return Crate(self.path, self.name, Crate.prerelease(self.version, prerelease), + self.dependencies.copy()) def next_version(self): - return Crate(self.path, self.name, str(semver.VersionInfo.parse(self.version).next_version('patch')), self.dependencies.copy()) + return Crate(self.path, self.name, + str(semver.VersionInfo.parse(self.version).next_version('patch')), + self.dependencies.copy()) def show_version(self): return self.version @@ -75,8 +87,8 @@ def show_version(self): def save(self, previous): contents = [] cargo_file = f"{self.path}/Cargo.toml" - with open(cargo_file, 'r') as r: - for line in r.readlines(): + with open(cargo_file, mode='r', encoding='utf-8') as cargo_file_read: + for line in cargo_file_read.readlines(): if line.startswith("version"): line = line.replace(previous.version, self.version) else: @@ -85,8 +97,8 @@ def save(self, previous): line = line.replace(previous.dependencies[dname], dversion) contents.append(line) - with open(cargo_file, 'w') as w: - w.write(''.join(contents)) + with open(cargo_file, mode='w', encoding='utf-8') as cargo_file_write: + cargo_file_write.write(''.join(contents)) def __str__(self): return f'Crate({self.path}, {self.name}, {self.version}, {self.dependencies})' @@ -94,7 +106,7 @@ def __str__(self): class Workspace: def __init__(self, crates): - names = set([c.name for c in crates]) + names = {c.name for c in crates} self.crates = {c.name: c.with_dependencies(names) for c in crates} def finalize_version(self): @@ -118,8 +130,8 @@ def next_version(self): return Workspace(Workspace.update_dependencies(crates).values()) def show_version(self): - for c in self.crates.values(): - return c.show_version() + for crate in self.crates.values(): + return crate.show_version() return "0.0.0" @classmethod @@ -133,16 +145,21 @@ def __str__(self): return f'Workspace({[str(c) for c in self.crates.values()]})' def save(self, previous): - for cn in self.crates.keys(): - self.crates[cn].save(previous.crates[cn]) + for crate_key in self.crates.keys(): + self.crates[crate_key].save(previous.crates[crate_key]) def load(root): - r = toml.load(f"{root}/Cargo.toml") - if "workspace" in r: - return Workspace([load(f"{root}/{path}") for path in r["workspace"]["members"]]) - - return Crate(path=root, name=r["package"]["name"], version=r["package"]["version"], dependencies={dn: r["dependencies"][dn]["version"] for dn in r["dependencies"] if "version" in r["dependencies"][dn]}) + root_cargo_file = toml.load(f"{root}/Cargo.toml") + if "workspace" in root_cargo_file: + return Workspace([load(f"{root}/{path}") + for path in root_cargo_file["workspace"]["members"]]) + + return Crate(path=root, name=root_cargo_file["package"]["name"], + version=root_cargo_file["package"]["version"], + dependencies={dn: root_cargo_file["dependencies"][dn]["version"] + for dn in root_cargo_file["dependencies"] + if "version" in root_cargo_file["dependencies"][dn]}) def parse_args(): diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..7bdcd8d0 --- /dev/null +++ b/renovate.json @@ -0,0 +1,31 @@ +{ + "extends": [ + "helpers:pinGitHubActionDigests" + ], + "labels": [ + "dependencies" + ], + "prCreation": "not-pending", + "reviewers": [ + "team:developers" + ], + "rollbackPrs": true, + "schedule": [ + "after 22:00 and before 6:00 every weekday" + ], + "timezone": "Europe/Berlin", + "packageRules": [ + { + "matchUpdateTypes": [ + "patch" + ], + "groupName": "All dependencies (patch only)" + } + ], + "lockFileMaintenance": { + "enabled": true, + "schedule": [ + "after 22:00 and before 6:00 every weekday" + ] + } +} diff --git a/scripts/generate-manifests.sh b/scripts/generate-manifests.sh index af3a0aff..17b09822 100755 --- a/scripts/generate-manifests.sh +++ b/scripts/generate-manifests.sh @@ -1,22 +1,22 @@ -#!/bin/bash +#!/usr/bin/env bash # This script reads a Helm chart from deploy/helm/hive-operator and # generates manifest files into deploy/manifestss set -e tmp=$(mktemp -d ./manifests-XXXXX) -helm template --output-dir $tmp \ +helm template --output-dir "$tmp" \ --include-crds \ --name-template hive-operator \ deploy/helm/hive-operator -for file in $(find $tmp -type f) +while IFS= read -r -d '' file do - yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' $file - yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' $file - sed -i '/# Source: .*/d' $file -done + yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' "$file" + yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' "$file" + sed -i '/# Source: .*/d' "$file" +done < <(find "$tmp" -type f) -cp -r $tmp/hive-operator/*/* deploy/manifests/ +cp -r "$tmp"/hive-operator/*/* deploy/manifests/ -rm -rf $tmp +rm -rf "$tmp"