diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 51de9d8b40..712dbfe283 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -18,8 +18,10 @@ jobs: - name: Apply Bump shell: bash + env: + NEW_VERSION: ${{ inputs.new_version }} run: | - bash ./scripts/bump_version.sh ${{ github.event.inputs.new_version }} + bash ./scripts/release/bump-version.sh "$NEW_VERSION" - name: Create Pull Request uses: peter-evans/create-pull-request@v4 diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 9cf2b3ebe8..ec665a95d1 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -2,22 +2,24 @@ name: Code Scanning Query Pack Generation on: merge_group: + types: [checks_requested] pull_request: branches: - main - - "rc/**" - next + - "rc/**" push: branches: - main - - "rc/**" - next + - "rc/**" env: XARGS_MAX_PROCS: 4 jobs: + prepare-code-scanning-pack-matrix: name: Prepare CodeQL Code Scanning pack matrix runs-on: ubuntu-22.04 @@ -25,24 +27,23 @@ jobs: matrix: ${{ steps.export-code-scanning-pack-matrix.outputs.matrix }} steps: - name: Checkout repository - uses: actions/checkout@v2 - + uses: actions/checkout@v4 - name: Export Code Scanning pack matrix id: export-code-scanning-pack-matrix run: | - echo "::set-output name=matrix::$( + echo "matrix=$( jq --compact-output '.supported_environment | {include: .}' supported_codeql_configs.json - )" + )" >> $GITHUB_OUTPUT create-code-scanning-pack: name: Create Code Scanning pack needs: prepare-code-scanning-pack-matrix - runs-on: ubuntu-20.04-xl + runs-on: ubuntu-latest-xl strategy: fail-fast: false matrix: ${{ fromJSON(needs.prepare-code-scanning-pack-matrix.outputs.matrix) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Cache CodeQL id: cache-codeql @@ -65,18 +66,28 @@ jobs: with: cli_path: ${{ github.workspace }}/codeql_home/codeql + - name: Determine ref for external help files + id: determine-ref + run: | + if [[ $GITHUB_EVENT_NAME == "pull_request" || $GITHUB_EVENT_NAME == "merge_group" ]]; then + echo "EXTERNAL_HELP_REF=$GITHUB_HEAD_REF" >> "$GITHUB_ENV" + else + echo "EXTERNAL_HELP_REF=$GITHUB_REF" >> "$GITHUB_ENV" + fi + echo "Using ref $EXTERNAL_HELP_REF for external help files." + - name: Checkout external help files continue-on-error: true id: checkout-external-help-files - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ssh-key: ${{ secrets.CODEQL_CODING_STANDARDS_HELP_KEY }} repository: "github/codeql-coding-standards-help" - ref: ${{ github.head_ref }} + ref: ${{ env.EXTERNAL_HELP_REF }} path: external-help-files - name: Include external help files - if: ${{ steps.checkout-external-help-files.outcome == 'success' }} + if: steps.checkout-external-help-files.outcome == 'success' run: | pushd external-help-files find . -name '*.md' -exec rsync -av --relative {} "$GITHUB_WORKSPACE" \; diff --git a/.github/workflows/codeql_unit_tests.yml b/.github/workflows/codeql_unit_tests.yml index 053bea4985..62660d973d 100644 --- a/.github/workflows/codeql_unit_tests.yml +++ b/.github/workflows/codeql_unit_tests.yml @@ -2,17 +2,20 @@ name: CodeQL Unit Testing on: merge_group: + types: [checks_requested] push: branches: - main - - "rc/**" - next + - "rc/**" pull_request: branches: - - "**" - workflow_dispatch: + - main + - next + - "rc/**" jobs: + prepare-unit-test-matrix: name: Prepare CodeQL unit test matrix runs-on: ubuntu-22.04 @@ -20,16 +23,16 @@ jobs: matrix: ${{ steps.export-unit-test-matrix.outputs.matrix }} steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Export unit test matrix id: export-unit-test-matrix run: | echo "Merging Result:" python scripts/create_language_matrix.py - echo "::set-output name=matrix::$( + echo "matrix=$( python scripts/create_language_matrix.py | \ - jq --compact-output 'map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')" + jq --compact-output 'map([.+{os: "ubuntu-latest-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')" >> $GITHUB_OUTPUT run-test-suites: name: Run unit tests @@ -39,22 +42,22 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.prepare-unit-test-matrix.outputs.matrix) }} - + steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Python uses: actions/setup-python@v4 with: python-version: "3.9" - + - name: Install Python dependencies run: pip install -r scripts/requirements.txt - name: Cache CodeQL id: cache-codeql - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: # A list of files, directories, and wildcard patterns to cache and restore path: ${{github.workspace}}/codeql_home @@ -101,7 +104,7 @@ jobs: def print_error(fmt, *args): print(f"::error::{fmt}", *args) - + def print_error_and_fail(fmt, *args): print_error(fmt, args) sys.exit(1) @@ -148,7 +151,7 @@ jobs: file.close() - name: Upload test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.language }}-test-results-${{ runner.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }} path: | @@ -157,11 +160,11 @@ jobs: validate-test-results: name: Validate test results - needs: [run-test-suites] + needs: run-test-suites runs-on: ubuntu-22.04 steps: - name: Collect test results - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Validate test results run: | diff --git a/.github/workflows/create-draft-release.yml b/.github/workflows/create-draft-release.yml deleted file mode 100644 index f2818b15ab..0000000000 --- a/.github/workflows/create-draft-release.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Create draft release - -on: - workflow_dispatch: - inputs: - release_version_tag: - description: | - The tag for the new draft release, e.g. 0.5.1 - do not include the `v`. - required: true - codeql_analysis_threads: - description: | - Number of threads to evaluate queries - required: true - default: 6 - aws_ec2_instance_type: - description: | - Recommended specs: 8+ vCPU 32+ GB RAM (e.g. t2.2xlarge, r5.2xlarge) - required: true - default: r5.2xlarge - -jobs: - create-draft-release: - name: Create draft release - runs-on: ubuntu-22.04 - env: - # AWS CONFIGURATION - AWS_EC2_INSTANCE_TYPE: ${{ github.event.inputs.aws_ec2_instance_type }} - - # CODEQL CONFIGURATION - CODEQL_ANALYSIS_THREADS: ${{ github.event.inputs.codeql_analysis_threads }} - - # INTEGRATION TESTING CONFIGURATION - INTEGRATION_TESTING_ACCESS_TOKEN: ${{ secrets.INTEGRATION_TESTING_ACCESS_TOKEN }} - WORKFLOW_ID: 11846210 - - # RELEASE VERSION TAG - RELEASE_VERSION_TAG: ${{ github.event.inputs.release_version_tag }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install generate_release_notes.py dependencies - run: pip install -r scripts/requirements.txt - - - name: Create draft release - run: | - scripts/release/create_draft_release.sh ${GITHUB_REF#refs/heads/} "$RELEASE_VERSION_TAG" - env: - GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/dispatch-matrix-check.yml b/.github/workflows/dispatch-matrix-check.yml index a1cf8606a1..350f2fb73f 100644 --- a/.github/workflows/dispatch-matrix-check.yml +++ b/.github/workflows/dispatch-matrix-check.yml @@ -9,12 +9,12 @@ on: jobs: dispatch-matrix-check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Test Variables shell: pwsh - run: | + run: | Write-Host "Running as: ${{github.actor}}" - name: Dispatch Matrix Testing Job diff --git a/.github/workflows/dispatch-matrix-test-on-comment.yml b/.github/workflows/dispatch-matrix-test-on-comment.yml index bb307864c6..bef0ba7232 100644 --- a/.github/workflows/dispatch-matrix-test-on-comment.yml +++ b/.github/workflows/dispatch-matrix-test-on-comment.yml @@ -11,12 +11,12 @@ on: jobs: dispatch-matrix-check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Test Variables shell: pwsh - run: | + run: | Write-Host "Running as: ${{github.actor}}" $actor = "${{github.actor}}" diff --git a/.github/workflows/dispatch-release-performance-check.yml b/.github/workflows/dispatch-release-performance-check.yml index abba5328bd..0858527721 100644 --- a/.github/workflows/dispatch-release-performance-check.yml +++ b/.github/workflows/dispatch-release-performance-check.yml @@ -10,12 +10,12 @@ on: jobs: dispatch-matrix-check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Test Variables shell: pwsh - run: | + run: | Write-Host "Running as: ${{github.actor}}" $actor = "${{github.actor}}" diff --git a/.github/workflows/extra-rule-validation.yml b/.github/workflows/extra-rule-validation.yml index 1b2c1a3aef..a18f47c65d 100644 --- a/.github/workflows/extra-rule-validation.yml +++ b/.github/workflows/extra-rule-validation.yml @@ -2,6 +2,7 @@ name: ⚙️ Extra Rule Validation on: merge_group: + types: [checks_requested] push: branches: - main diff --git a/.github/workflows/finalize-release.yml b/.github/workflows/finalize-release.yml new file mode 100644 index 0000000000..fbadfdb836 --- /dev/null +++ b/.github/workflows/finalize-release.yml @@ -0,0 +1,86 @@ +name: Finalize Release +on: + pull_request: + types: + - closed + branches: + - "rc/**" + workflow_dispatch: + inputs: + ref: + description: | + The release branch to finalize. + required: true + +jobs: + finalize-release: + if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-22.04 + steps: + - name: Determine ref + env: + REF_FROM_INPUT: ${{ inputs.ref }} + REF_FROM_PR: ${{ github.event.pull_request.merge_commit_sha }} + BASE_REF_FROM_PR: ${{ github.event.pull_request.base.ref }} + run: | + if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" ]]; then + echo "REF=$REF_FROM_INPUT" >> "$GITHUB_ENV" + echo "BASE_REF=$REF_FROM_INPUT" >> "$GITHUB_ENV" + else + echo "REF=$REF_FROM_PR" >> "$GITHUB_ENV" + echo "BASE_REF=$BASE_REF_FROM_PR" >> "$GITHUB_ENV" + fi + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.REF }} + + - name: Configure git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Update release tag + run: | + version=${BASE_REF#rc/} + echo "Creating release tag v$version" + + git tag -a v$version -m "Release v$version" + git push -f origin v$version + + - name: Finalize release + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + version=${BASE_REF#rc/} + echo "Finalizing release v$version" + + gh release edit "v$version" --draft=false --tag=v$version + + - name: Determine if release was a hotfix release + run: | + version=${BASE_REF#rc/} + echo "HOTFIX_RELEASE=$(python scripts/release/is-hotfix.py $version)" >> "$GITHUB_ENV" + + - name: Bump main version + if: env.HOTFIX_RELEASE == 'false' + env: + GH_TOKEN: ${{ github.token }} + run: | + version=${BASE_REF#rc/} + next_version="$version-dev" + echo "Bumping main version to $next_version" + + git switch main + git pull --ff-only origin main + + git switch -c release-automation/bump-version + + ./scripts/release/bump-version.sh "$next_version" + + git add -u . + git commit -m "Bump version to $next_version" + git push --set-upstream origin release-automation/bump-version + + gh pr create --repo $GITHUB_REPOSITORY --base main --head release-automation/bump-version --body "Bump the version of main to the dev label of the just released version $next_version" --title "Bump version to $next_version" diff --git a/.github/workflows/generate-html-docs.yml b/.github/workflows/generate-html-docs.yml index bb12ba8a2b..f8e3d6d30c 100644 --- a/.github/workflows/generate-html-docs.yml +++ b/.github/workflows/generate-html-docs.yml @@ -2,6 +2,7 @@ name: Generate HTML documentation on: merge_group: + types: [checks_requested] push: branches: - main diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000000..9bbd27ce26 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,166 @@ +name: "Prepare CodeQL Coding Standards release" + +on: + workflow_dispatch: + inputs: + version: + description: | + The version to release (MUST follow semantic versioning so NO 'v' prefix). + required: true + ref: + description: | + The git commit, branch, or tag to release from. + required: true + hotfix: + description: | + Hotfix release. + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + actions: write + checks: write + +env: + RELEASE_VERSION: ${{ inputs.version }} + HOTFIX_RELEASE: ${{ inputs.hotfix }} + +jobs: + prepare-release: + name: "Prepare release" + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install release script dependencies + run: pip install -r scripts/release/requirements.txt + + - name: Validate version + run: | + if [[ "$HOTFIX_RELEASE" == "true" ]]; then + python scripts/release/validate-version.py --hotfix "$RELEASE_VERSION" + else + python scripts/release/validate-version.py "$RELEASE_VERSION" + fi + + - name: Check if release exists + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + release=$( { gh release view "v$RELEASE_VERSION" --json name,isDraft; } || echo "" ) + if [[ -z "$release" ]]; then + echo "Release v$RELEASE_VERSION does not exist. Proceeding" + echo "create_draft_release=true" >> "$GITHUB_ENV" + else + isDraft=$(echo "$release" | jq -r '.isDraft') + if [[ "$isDraft" != "true" ]]; then + echo "Release 'v$RELEASE_VERSION' already exists and is not a draft. Cannot proceed" + exit 1 + else + echo "Release 'v$RELEASE_VERSION' already exists and is a draft. Proceeding" + echo "create_draft_release=false" >> "$GITHUB_ENV" + fi + fi + + - name: Check if release PR exists + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + release_pr=$( { gh pr view "rc/$RELEASE_VERSION" --json title,state,number; } || echo "") + if [[ ! -z "$release_pr" ]]; then + pr_title=$(echo "$release_pr" | jq -r '.title') + pr_state=$(echo "$release_pr" | jq -r '.state') + pr_number=$(echo "$release_pr" | jq -r '.number') + echo "Found PR '$pr_title' with state '$pr_state'" + if [[ "$pr_title" == "Release v$RELEASE_VERSION" ]] && [[ "$pr_state" != "CLOSED" ]]; then + echo "Release PR is not closed, deleting it to proceed" + gh pr close --delete-branch $pr_number + fi + fi + + - name: Delete existing release branch + run: | + if [[ ! -z $(git ls-remote --heads origin rc/$RELEASE_VERSION) ]]; then + echo "Deleting existing release branch" + git push origin --delete rc/$RELEASE_VERSION + fi + + - name: Delete existing feature branch + run: | + if [[ ! -z $(git ls-remote --heads origin feature/update-user-manual-for-$RELEASE_VERSION) ]]; then + echo "Deleting existing feature branch" + git push origin --delete feature/update-user-manual-for-$RELEASE_VERSION + fi + + - name: Configure git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Create release branch + run: | + git switch -c rc/$RELEASE_VERSION + git push --set-upstream origin rc/$RELEASE_VERSION + + - name: Create draft release + if: env.create_draft_release == 'true' + env: + RELEASE_VERSION: ${{ inputs.version }} + GITHUB_TOKEN: ${{ github.token }} + run: | + # Create lightweight tag to reference release + git tag v$RELEASE_VERSION + git push -f origin v$RELEASE_VERSION + + gh release create \ + -R $GITHUB_REPOSITORY \ + --title "v$RELEASE_VERSION" \ + --draft \ + --target rc/$RELEASE_VERSION \ + v$RELEASE_VERSION + + - name: Create feature branch for PR + run: | + git switch -c feature/update-user-manual-for-$RELEASE_VERSION + git push --set-upstream origin feature/update-user-manual-for-$RELEASE_VERSION + + scripts/release/bump-version.sh "$RELEASE_VERSION" + + git add -u . + git commit -m "Update version" + git push + + - name: Generate token + id: generate-token + uses: actions/create-github-app-token@eaddb9eb7e4226c68cf4b39f167c83e5bd132b3e + with: + app-id: ${{ vars.AUTOMATION_APP_ID }} + private-key: ${{ secrets.AUTOMATION_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "codeql-coding-standards" + + - name: Create release PR + env: + # Use the token from the `generate-token` step because we can't use the default workflow token + # to create a PR and generate PR events to trigger the next workflow because of recursive workflow + # trigger protection. + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + gh pr create \ + -R $GITHUB_REPOSITORY \ + --title "Release v$RELEASE_VERSION" \ + --body "This PR releases codeql-coding-standards version $RELEASE_VERSION." \ + --base rc/$RELEASE_VERSION \ + --head feature/update-user-manual-for-$RELEASE_VERSION \ + --draft diff --git a/.github/workflows/tooling-unit-tests.yml b/.github/workflows/tooling-unit-tests.yml index 840e7c5b97..333b4ce024 100644 --- a/.github/workflows/tooling-unit-tests.yml +++ b/.github/workflows/tooling-unit-tests.yml @@ -2,6 +2,7 @@ name: 🧰 Tooling unit tests on: merge_group: + types: [checks_requested] push: branches: - main @@ -16,7 +17,7 @@ on: jobs: prepare-supported-codeql-env-matrix: name: Prepare supported CodeQL environment matrix - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.export-supported-codeql-env-matrix.outputs.matrix }} steps: @@ -33,7 +34,7 @@ jobs: analysis-report-tests: name: Run analysis report tests needs: prepare-supported-codeql-env-matrix - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: ${{ fromJSON(needs.prepare-supported-codeql-env-matrix.outputs.matrix) }} @@ -79,7 +80,7 @@ jobs: recategorization-tests: name: Run Guideline Recategorization tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/update-check-run.yml b/.github/workflows/update-check-run.yml new file mode 100644 index 0000000000..225c81fa24 --- /dev/null +++ b/.github/workflows/update-check-run.yml @@ -0,0 +1,70 @@ +name: Update check run + +on: + workflow_dispatch: + inputs: + id: + description: | + The unique identifier of the check run. + type: number + required: true + status: + description: | + The current status. + + Can be one of: queued, in_progress, completed + type: string + required: true + conclusion: + description: | + The final conclusion of the check when completed. + + Can be one of: action_required, cancelled, failure, neutral, success, skipped, stale, timed_out + type: string + details_url: + description: | + The URL of the integrator's site that has the full details of the check. + type: string + external_id: + description: | + A reference for the run on the integrator's system. + type: string + output: + description: | + The output object for the check run. + + See https://docs.github.com/en/rest/checks/runs?apiVersion=2022-11-28#update-a-check-run for more information. + type: string + default: '{}' + +permissions: + checks: write + +jobs: + update-check-run: + runs-on: ubuntu-22.04 + steps: + - name: Update check run + env: + CHECK_RUN_ID: ${{ inputs.id }} + CHECK_RUN_STATUS: ${{ inputs.status }} + CHECK_RUN_CONCLUSION: ${{ inputs.conclusion }} + CHECK_RUN_DETAILS_URL: ${{ inputs.details_url }} + CHECK_RUN_EXTERNAL_ID: ${{ inputs.external_id }} + CHECK_RUN_OUTPUT: ${{ inputs.output }} + GITHUB_TOKEN: ${{ github.token }} + run: | + jq -n \ + --arg status "$CHECK_RUN_STATUS" \ + --arg conclusion "$CHECK_RUN_CONCLUSION" \ + --arg details_url "$CHECK_RUN_DETAILS_URL" \ + --arg external_id "$CHECK_RUN_EXTERNAL_ID" \ + --argjson output "$CHECK_RUN_OUTPUT" \ + '{status: $status, conclusion: $conclusion, details_url: $details_url, external_id: $external_id, output: $output} | to_entries | map(select(.value != "" and .value != {})) | from_entries' \ + | \ + gh api \ + --method PATCH \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --input - \ + /repos/$GITHUB_REPOSITORY/check-runs/$CHECK_RUN_ID diff --git a/.github/workflows/update-release-status.yml b/.github/workflows/update-release-status.yml new file mode 100644 index 0000000000..15e212f369 --- /dev/null +++ b/.github/workflows/update-release-status.yml @@ -0,0 +1,144 @@ +name: "Update Release Status" +on: + check_run: + types: + - completed + - rerequested + branches: + - "rc/**" + + workflow_dispatch: + inputs: + head-sha: + description: | + The head SHA to use. + type: string + required: true + +permissions: + actions: write + checks: write + contents: write + +jobs: + validate-check-runs: + runs-on: ubuntu-22.04 + outputs: + status: ${{ steps.set-output.outputs.status }} + check-run-head-sha: ${{ steps.set-output.outputs.check-run-head-sha }} + steps: + - name: Determine check run head SHA + env: + HEAD_SHA_FROM_EVENT: ${{ github.event.check_run.head_sha }} + HEAD_SHA_FROM_INPUTS: ${{ inputs.head-sha }} + run: | + if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" ]]; then + echo "CHECK_RUN_HEAD_SHA=$HEAD_SHA_FROM_INPUTS" >> "$GITHUB_ENV" + else + echo "CHECK_RUN_HEAD_SHA=$HEAD_SHA_FROM_EVENT" >> "$GITHUB_ENV" + fi + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.CHECK_RUN_HEAD_SHA }} + + - name: Get release status check run + id: get-check-run + if: (github.event_name == 'check_run' && github.event.check_run.conclusion == 'success' && github.event.check_run.name != github.workflow) || github.event_name == 'workflow_dispatch' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + check_run_info=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --jq '.check_runs[] | select(.name == "release-status") | {id: .id, status: .status, conclusion: .conclusion}' \ + /repos/$GITHUB_REPOSITORY/commits/$CHECK_RUN_HEAD_SHA/check-runs) + + check_run_id=$(echo "$check_run_info" | jq -r '.id') + check_run_status=$(echo "$check_run_info" | jq -r '.status') + check_run_conclusion=$(echo "$check_run_info" | jq -r '.conclusion') + + echo "CHECK_RUN_ID=$check_run_id" >> "$GITHUB_ENV" + echo "CHECK_RUN_STATUS=$check_run_status" >> "$GITHUB_ENV" + echo "CHECK_RUN_CONCLUSION=$check_run_conclusion" >> "$GITHUB_ENV" + + - name: Reset release status + if: env.CHECK_RUN_STATUS == 'completed' && ((github.event_name == 'check_run' && github.event.action == 'rerequested') || github.event_name == 'workflow_dispatch') + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + CHECK_RUN_ID=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --field name="release-status" \ + --field head_sha="$CHECK_RUN_HEAD_SHA" \ + --jq ".id" \ + /repos/$GITHUB_REPOSITORY/check-runs) + + echo "Created release status check run with id $CHECK_RUN_ID" + + - name: Check all runs completed + if: env.CHECK_RUN_STATUS != 'completed' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + check_runs=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --jq '.check_runs | map(select(.name != "release-status"))' \ + /repos/$GITHUB_REPOSITORY/commits/$CHECK_RUN_HEAD_SHA/check-runs) + + status_stats=$(echo "$check_runs" | jq -r '. | {failed: (map(select(.conclusion == "failure")) | length), pending: (map(select(.status != "completed")) | length) }') + + failed=$(echo "$status_stats" | jq -r '.failed') + pending=$(echo "$status_stats" | jq -r '.pending') + + echo "CHECK_RUNS_FAILED=$failed" >> "$GITHUB_ENV" + echo "CHECK_RUNS_PENDING=$pending" >> "$GITHUB_ENV" + + - name: Conclude release status + if: env.CHECK_RUNS_PENDING == '0' && env.CHECK_RUN_STATUS != 'completed' + env: + GITHUB_TOKEN: ${{ github.token }} + CHECK_RUNS_FAILED: ${{ env.check-runs-failed }} + run: | + if [[ "$CHECK_RUNS_FAILED" == "0" ]]; then + echo "All check runs succeeded" + conclusion="success" + else + echo "Some check runs failed" + conclusion="failure" + fi + + jq -n \ + --arg status "completed" \ + --arg conclusion "$conclusion" \ + '{status: $status, conclusion: $conclusion}' \ + | \ + gh api \ + --method PATCH \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --input - \ + /repos/$GITHUB_REPOSITORY/check-runs/$CHECK_RUN_ID + + - name: Set output + id: set-output + run: | + if [[ "$CHECK_RUNS_PENDING" == "0" ]]; then + echo "status=completed" >> "$GITHUB_OUTPUT" + else + echo "status=in_progress" >> "$GITHUB_OUTPUT" + fi + + echo "check-run-head-sha=$CHECK_RUN_HEAD_SHA" >> "$GITHUB_OUTPUT" + + update-release: + needs: validate-check-runs + if: needs.validate-check-runs.outputs.status == 'completed' + uses: ./.github/workflows/update-release.yml + with: + head-sha: ${{ needs.validate-check-runs.outputs.check-run-head-sha }} + secrets: + AUTOMATION_PRIVATE_KEY: ${{ secrets.AUTOMATION_PRIVATE_KEY }} diff --git a/.github/workflows/update-release.yml b/.github/workflows/update-release.yml new file mode 100644 index 0000000000..9a7d95c846 --- /dev/null +++ b/.github/workflows/update-release.yml @@ -0,0 +1,72 @@ +name: Update Release + +on: + workflow_dispatch: + inputs: + head-sha: + description: | + The head SHA of the release PR to use for finalizing the release. + required: true + workflow_call: + inputs: + head-sha: + type: string + description: | + The head SHA of the release PR to use for finalizing the release. + required: true + secrets: + AUTOMATION_PRIVATE_KEY: + description: | + The private key to use to generate a token for accessing the release engineering repository. + required: true +env: + HEAD_SHA: ${{ inputs.head-sha }} + +jobs: + update-release: + name: "Update release" + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.head-sha }} + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install dependencies + run: pip install -r scripts/release/requirements.txt + + - name: Generate token + id: generate-token + uses: actions/create-github-app-token@eaddb9eb7e4226c68cf4b39f167c83e5bd132b3e + with: + app-id: ${{ vars.AUTOMATION_APP_ID }} + private-key: ${{ secrets.AUTOMATION_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "codeql-coding-standards-release-engineering" + + - name: Update release assets + env: + GITHUB_TOKEN: ${{ github.token }} + RELEASE_ENGINEERING_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + python scripts/release/update-release-assets.py \ + --head-sha $HEAD_SHA \ + --layout scripts/release/release-layout.yml \ + --repo "$GITHUB_REPOSITORY" \ + --github-token "$GITHUB_REPOSITORY:$GITHUB_TOKEN" "github/codeql-coding-standards-release-engineering:$RELEASE_ENGINEERING_TOKEN" \ + --skip-checkrun "release-status" \ + --skip-checks + + - name: Update release notes + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python scripts/release/update-release-notes.py \ + --head-sha $HEAD_SHA \ + --repo "$GITHUB_REPOSITORY" \ + --github-token "$GITHUB_TOKEN" diff --git a/.github/workflows/validate-coding-standards.yml b/.github/workflows/validate-coding-standards.yml deleted file mode 100644 index 84b91f5280..0000000000 --- a/.github/workflows/validate-coding-standards.yml +++ /dev/null @@ -1,185 +0,0 @@ -name: Validating Coding Standards - -on: - merge_group: - push: - branches: - - main - - "rc/**" - - next - pull_request: - branches: - - main - - "rc/**" - - next - -env: - XARGS_MAX_PROCS: 4 - -jobs: - validate-package-files: - name: Validate Package Files - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install CodeQL - run: | - VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)" - gh extensions install github/gh-codeql - gh codeql set-version "$VERSION" - gh codeql install-stub - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: Install generate_package_files.py dependencies - run: pip install -r scripts/requirements.txt - - - name: Validate Package Descriptions (CPP) - run: | - python scripts/validate-rule-package.py rule_packages/cpp/*.json - - - name: Validate Package Descriptions (C) - run: | - python scripts/validate-rule-package.py rule_packages/c/*.json - - - name: Validate Package Descriptions consistency (CPP) - run: | - python scripts/verify_rule_package_consistency.py cpp - - - name: Validate Package Descriptions consistency (C) - run: | - python scripts/verify_rule_package_consistency.py c - - - name: Validate Package Files (CPP) - run: | - find rule_packages/cpp -name \*.json -exec basename {} .json \; | xargs python scripts/generate_rules/generate_package_files.py cpp - git diff - git diff --compact-summary - git diff --quiet - - - name: Validate Package Files (C) - run: | - find rule_packages/c -name \*.json -exec basename {} .json \; | xargs python scripts/generate_rules/generate_package_files.py c - git diff - git diff --compact-summary - git diff --quiet - - validate-codeql-format: - name: "Validate CodeQL Format" - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install CodeQL - run: | - VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)" - gh extensions install github/gh-codeql - gh codeql set-version "$VERSION" - gh codeql install-stub - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: Validate CodeQL Format (CPP) - run: | - find cpp \( -name \*.ql -or -name \*.qll \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place - - git diff - git diff --compact-summary - git diff --quiet - - - name: Validate CodeQL Format (C) - run: | - find c \( -name \*.ql -or -name \*.qll \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place - - git diff - git diff --compact-summary - git diff --quiet - - validate-query-help-files: - name: Validate Query Help Files - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Validate CPP Query Help Files - run: | - exit_code=0 - for help_file in `find cpp -name '*.md'` - do - if grep -F -q 'REPLACE THIS' "$help_file" > /dev/null - then - echo "Help file $help_file contains placeholders that are not replaced or removed!" - exit_code=1 - fi - done - - exit $exit_code - - - name: Validate C Query Help Files - run: | - exit_code=0 - for help_file in `find c -name '*.md'` - do - if grep -F -q 'REPLACE THIS' "$help_file" > /dev/null - then - echo "Help file $help_file contains placeholders that are not replaced or removed!" - exit_code=1 - fi - done - - exit $exit_code - - validate-cpp-test-files: - name: Validate C++ Test Files - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install clang-format - run: | - sudo apt-get install --yes --quiet --no-install-recommends clang-format - echo "::debug::$(clang-format -version)" - - - name: Validate C++ Test Files - run: | - if ! test -f .clang-format; then - echo "Cannot find .clang-format in '$PWD'. Exiting..." - fi - - find cpp/*/test -name \*.cpp -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" clang-format --style=file -i --verbose - git diff - git diff --compact-summary - git diff --quiet - - validate-c-test-files: - name: Validate C Test Files - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install clang-format - run: | - sudo apt-get install --yes --quiet --no-install-recommends clang-format - echo "::debug::$(clang-format -version)" - - - name: Validate C++ Test Files - run: | - if ! test -f .clang-format; then - echo "Cannot find .clang-format in '$PWD'. Exiting..." - fi - - find c/*/test -name \*.c -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" clang-format --style=file -i --verbose - git diff - git diff --compact-summary - git diff --quiet diff --git a/.github/workflows/validate-package-files.yml b/.github/workflows/validate-package-files.yml new file mode 100644 index 0000000000..0573b00590 --- /dev/null +++ b/.github/workflows/validate-package-files.yml @@ -0,0 +1,59 @@ +name: Validate Package Files +on: + merge_group: + types: [checks_requested] + pull_request: + branches: + - main + - next + - "rc/**" + +jobs: + validate-package-files: + strategy: + matrix: + language: [cpp, c] + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install CodeQL + run: | + VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)" + gh extensions install github/gh-codeql + gh codeql set-version "$VERSION" + gh codeql install-stub + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Install generate_package_files.py dependencies + run: pip install -r scripts/requirements.txt + + - name: Validate Package Descriptions + env: + LANGUAGE: ${{ matrix.language }} + run: | + python scripts/validate-rule-package.py rule_packages/$LANGUAGE/*.json + + - name: Validate Package Descriptions consistency + env: + LANGUAGE: ${{ matrix.language }} + run: | + python scripts/verify_rule_package_consistency.py $LANGUAGE + + - name: Validate Current versus Expected Package Files + env: + LANGUAGE: ${{ matrix.language }} + run: | + find rule_packages/$LANGUAGE -name \*.json -exec basename {} .json \; | xargs python scripts/generate_rules/generate_package_files.py $LANGUAGE + git diff + git diff --compact-summary + git diff --quiet \ No newline at end of file diff --git a/.github/workflows/validate-query-formatting.yml b/.github/workflows/validate-query-formatting.yml new file mode 100644 index 0000000000..e4c6871ad5 --- /dev/null +++ b/.github/workflows/validate-query-formatting.yml @@ -0,0 +1,44 @@ +name: "Validate Query Formatting" +on: + merge_group: + types: [checks_requested] + pull_request: + branches: + - main + - next + - "rc/**" + +env: + XARGS_MAX_PROCS: 4 + +jobs: + validate-query-formatting: + strategy: + matrix: + language: [cpp, c] + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Install CodeQL + run: | + VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)" + gh extensions install github/gh-codeql + gh codeql set-version "$VERSION" + gh codeql install-stub + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Validate query format + env: + LANGUAGE: ${{ matrix.language }} + run: | + codeql version + find $LANGUAGE \( -name \*.ql -or -name \*.qll \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place + + git diff + git diff --compact-summary + git diff --quiet \ No newline at end of file diff --git a/.github/workflows/validate-query-help.yml b/.github/workflows/validate-query-help.yml new file mode 100644 index 0000000000..d99144fc7f --- /dev/null +++ b/.github/workflows/validate-query-help.yml @@ -0,0 +1,37 @@ +name: Validate Query Help Files +on: + merge_group: + types: [checks_requested] + pull_request: + branches: + - main + - next + - "rc/**" + +jobs: + validate-query-help-files: + strategy: + matrix: + language: [cpp, c] + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Validate Query Help Files + env: + LANGUAGE: ${{ matrix.language }} + run: | + exit_code=0 + for help_file in `find $LANGUAGE -name '*.md'` + do + if grep -F -q 'REPLACE THIS' "$help_file" > /dev/null + then + echo "Help file $help_file contains placeholders that are not replaced or removed!" + exit_code=1 + fi + done + + exit $exit_code \ No newline at end of file diff --git a/.github/workflows/validate-query-test-case-formatting.yml b/.github/workflows/validate-query-test-case-formatting.yml new file mode 100644 index 0000000000..7b95484376 --- /dev/null +++ b/.github/workflows/validate-query-test-case-formatting.yml @@ -0,0 +1,45 @@ +name: Validate Query Test Case Formatting +on: + merge_group: + types: [checks_requested] + pull_request: + branches: + - main + - next + - "rc/**" + +env: + XARGS_MAX_PROCS: 4 + +jobs: + validate-test-case-files: + runs-on: ubuntu-22.04 + strategy: + matrix: + language: [cpp, c] + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Install clang-format + run: | + sudo apt-get install --yes --quiet --no-install-recommends clang-format + + - name: Validating Current versus Expected Test Case Formatting + env: + LANGUAGE: ${{ matrix.language }} + # IMPORTANT: This step current relies on the fact that a file extension is the same as the language name for simplicity. + run: | + if ! test -f .clang-format; then + echo "Cannot find .clang-format in '$PWD'. Exiting..." + fi + + find $LANGUAGE/*/test -name \*.$LANGUAGE -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" clang-format --style=file -i --verbose + git diff + git diff --compact-summary + git diff --quiet + + \ No newline at end of file diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml new file mode 100644 index 0000000000..5f5382f5dd --- /dev/null +++ b/.github/workflows/validate-release.yml @@ -0,0 +1,171 @@ +name: Validate release + +on: + pull_request: + branches: + - "rc/**" + +permissions: + contents: read + actions: write + checks: write + +env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + +jobs: + pre-validate-performance: + outputs: + check-run-id: ${{ steps.create-check-run.outputs.check-run-id }} + runs-on: ubuntu-22.04 + steps: + - name: Create check run + id: create-check-run + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + check_run_id=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --field name="performance-test" \ + --field head_sha="$HEAD_SHA" \ + --jq ".id" \ + /repos/$GITHUB_REPOSITORY/check-runs) + + echo "check-run-id=$check_run_id" >> "$GITHUB_OUTPUT" + + validate-performance: + needs: pre-validate-performance + runs-on: ubuntu-22.04 + steps: + - name: Generate token + id: generate-token + uses: actions/create-github-app-token@eaddb9eb7e4226c68cf4b39f167c83e5bd132b3e + with: + app-id: ${{ vars.AUTOMATION_APP_ID }} + private-key: ${{ secrets.AUTOMATION_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "codeql-coding-standards-release-engineering" + - name: Invoke performance test + env: + CHECK_RUN_ID: ${{ needs.pre-validate-performance.outputs.check-run-id }} + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + jq -n \ + --arg ref "$HEAD_SHA" \ + --arg check_run_id "$CHECK_RUN_ID" \ + '{ref: $ref, "check-run-id": $check_run_id}' \ + | \ + gh workflow run release-performance-testing.yml \ + --json \ + -R github/codeql-coding-standards-release-engineering + + on-failure-validate-performance-dispatch: + needs: [pre-validate-performance, validate-performance] + if: failure() + runs-on: ubuntu-22.04 + steps: + - name: Fail check run status + env: + CHECK_RUN_ID: ${{ needs.pre-validate-performance.outputs.check-run-id }} + GITHUB_TOKEN: ${{ github.token }} + run: | + jq -n \ + --arg status "completed" \ + --arg conclusion "failure" \ + '{status: $status, conclusion: $conclusion}' \ + | \ + gh api \ + --method PATCH \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --input - \ + /repos/$GITHUB_REPOSITORY/check-runs/$CHECK_RUN_ID + + pre-validate-compiler-compatibility: + outputs: + check-run-id: ${{ steps.create-check-run.outputs.check-run-id }} + runs-on: ubuntu-22.04 + steps: + - name: Create check run + id: create-check-run + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + check_run_id=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --field name="compiler-compatibility-test" \ + --field head_sha="$HEAD_SHA" \ + --jq ".id" \ + /repos/$GITHUB_REPOSITORY/check-runs) + + echo "check-run-id=$check_run_id" >> "$GITHUB_OUTPUT" + + validate-compiler-compatibility: + needs: pre-validate-compiler-compatibility + runs-on: ubuntu-22.04 + steps: + - name: Generate token + id: generate-token + uses: actions/create-github-app-token@eaddb9eb7e4226c68cf4b39f167c83e5bd132b3e + with: + app-id: ${{ vars.AUTOMATION_APP_ID }} + private-key: ${{ secrets.AUTOMATION_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "codeql-coding-standards-release-engineering" + - name: Invoke compiler compatibility test + env: + CHECK_RUN_ID: ${{ needs.pre-validate-compiler-compatibility.outputs.check-run-id }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + jq -n \ + --arg ref "$HEAD_SHA" \ + --arg check_run_id "$CHECK_RUN_ID" \ + '{ref: $ref, "check-run-id": $check_run_id}' \ + | \ + gh workflow run release-compiler-validation.yml \ + --json \ + -R github/codeql-coding-standards-release-engineering + + on-failure-validate-compiler-compatibility-dispatch: + needs: + [pre-validate-compiler-compatibility, validate-compiler-compatibility] + if: failure() + runs-on: ubuntu-22.04 + steps: + - name: Fail check run status + env: + CHECK_RUN_ID: ${{ needs.pre-validate-compiler-compatibility.outputs.check-run-id }} + GITHUB_TOKEN: ${{ github.token }} + run: | + jq -n \ + --arg status "completed" \ + --arg conclusion "failure" \ + '{status: $status, conclusion: $conclusion}' \ + | \ + gh api \ + --method PATCH \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --input - \ + /repos/$GITHUB_REPOSITORY/check-runs/$CHECK_RUN_ID + + create-release-status-check-run: + name: "Initialize release status monitoring" + runs-on: ubuntu-22.04 + steps: + - name: Create release status check run + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + CHECK_RUN_ID=$(gh api \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --field name="release-status" \ + --field head_sha="$HEAD_SHA" \ + --field status="in_progress" \ + --jq ".id" \ + /repos/$GITHUB_REPOSITORY/check-runs) + + echo "Created release status check run with id $CHECK_RUN_ID for $GITHUB_SHA" diff --git a/.github/workflows/verify-standard-library-dependencies.yml b/.github/workflows/verify-standard-library-dependencies.yml index ab78744e4e..cd5d35248d 100644 --- a/.github/workflows/verify-standard-library-dependencies.yml +++ b/.github/workflows/verify-standard-library-dependencies.yml @@ -3,6 +3,7 @@ name: Verify Standard Library Dependencies # Run this workflow every time the "supported_codeql_configs.json" file or a "qlpack.yml" file is changed on: merge_group: + types: [checks_requested] pull_request: branches: - main @@ -16,7 +17,7 @@ on: jobs: prepare-matrix: name: Prepare CodeQL configuration matrix - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.export-matrix.outputs.matrix }} steps: diff --git a/change_notes/2023-10-10-add-certification-kit.md b/change_notes/2023-10-10-add-certification-kit.md new file mode 100644 index 0000000000..d143eaa61b --- /dev/null +++ b/change_notes/2023-10-10-add-certification-kit.md @@ -0,0 +1 @@ +- The release artifacts now include a certification kit used for ISO26262 certification. \ No newline at end of file diff --git a/docs/development_handbook.md b/docs/development_handbook.md index 8aeb1ee5e5..2168e1fc56 100644 --- a/docs/development_handbook.md +++ b/docs/development_handbook.md @@ -35,10 +35,13 @@ | 0.26.0 | 2022-08-10 | Remco Vermeulen | Address incorrect package file generation command. This was missing the required language argument. | | 0.27.0 | 2022-11-08 | Luke Cartey | Update the versions of C we intend to support to exclude C90, which reflects the intended scope at the outset of the project. | | 0.28.0 | 2023-08-14 | Luke Cartey | Remove references to LGTM which is now a legacy product. | +| 0.29.0 | 2023-10-11 | Remco Vermeulen | Update release process. | +| 0.29.1 | 2023-10-11 | Remco Vermeulen | Address Markdown linter problems. | +| 0.30.0 | 2023-11-14 | Remco Vermeulen | Clarify release steps in case of a hotfix release. | ## Scope of work -A _coding standard_ is a set of rules or guidelines which restrict or prohibit the use of certain dangerous or confusing coding patterns or language features. This repository contains CodeQL queries (and supporting processes) which implement a number of different coding standards. The currently supported standards are: +A *coding standard* is a set of rules or guidelines which restrict or prohibit the use of certain dangerous or confusing coding patterns or language features. This repository contains CodeQL queries (and supporting processes) which implement a number of different coding standards. The currently supported standards are: | Standard | Version | Total rules | Total supportable rules | Status | Notes | | -------------------------------------------------------------------------------------------------------------------- | ------- | ----------- | ----------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -47,8 +50,7 @@ A _coding standard_ is a set of rules or guidelines which restrict or prohibit t | [CERT-C](https://resources.sei.cmu.edu/downloads/secure-coding/assets/sei-cert-c-coding-standard-2016-v01.pdf) | 2016 | 120 | 99 | In development | The implementation excludes rules not part of 2016, but that are added to the [CERT-C wiki](https://wiki.sei.cmu.edu/confluence/display/c/) | | [MISRA C](https://www.misra.org.uk/product/misra-c2012-third-edition-first-revision/ ) | 2012 | 172 | 172 | In development | This includes the [MISRA C:2012 Amendment 2](https://www.misra.org.uk/app/uploads/2021/06/MISRA-C-2012-AMD2.pdf) | - -Each coding standard consists of a list of "guidelines", however not all the guidelines in all the standards will be amenable to automated static analysis. The AUTOSAR C++ standard categorizes the guidelines according to enforcement by static analysis tools in section _5.1.3 Rule classification according to enforcement by static analysis_ of the standard. The CERT-C++ standard does not provide such categorization, but frequently has a [documented](https://wiki.sei.cmu.edu/confluence/display/cplusplus/How+this+Coding+Standard+Is+Organized#HowthisCodingStandardIsOrganized-AutomatedDetection) automated detection section for guidelines that documents tools, including their limitations, that can verify the guidelines in question. We have therefore carefully reviewed each supported standard. For each guidelines that is not categorized as automatic enforceable we have determined,in conjunction with end users, what parts of the guideline can be supported in which capacity with CodeQL. +Each coding standard consists of a list of "guidelines", however not all the guidelines in all the standards will be amenable to automated static analysis. The AUTOSAR C++ standard categorizes the guidelines according to enforcement by static analysis tools in section *5.1.3 Rule classification according to enforcement by static analysis* of the standard. The CERT-C++ standard does not provide such categorization, but frequently has a [documented](https://wiki.sei.cmu.edu/confluence/display/cplusplus/How+this+Coding+Standard+Is+Organized#HowthisCodingStandardIsOrganized-AutomatedDetection) automated detection section for guidelines that documents tools, including their limitations, that can verify the guidelines in question. We have therefore carefully reviewed each supported standard. For each guidelines that is not categorized as automatic enforceable we have determined,in conjunction with end users, what parts of the guideline can be supported in which capacity with CodeQL. For some of the rules which are not amenable to static analysis, we may opt to provide a query which aids with "auditing" the rules. For example, AUTOSAR includes a rule (A10-0-1) "Public inheritance shall be used to implement 'is-a' relationship.". This is not directly amenable to static analysis, because it requires external context around the concept being modeled. However, we can provide an "audit" rule which reports all the public and private inheritance relationships in the program, so they can be manually verified. @@ -62,8 +64,8 @@ A common use case for the coding standards specified above is to to help in the To support the functional safety use case, the scope of work for this project also includes: - - _Analysis reporting_ - producing reports for functional safety purposes that summarize the findings and highlight any issues during analysis that could compromise the integrity of those findings. - - _Deviations_ - a process for suppressing valid results, and maintaining metadata +- *Analysis reporting* - producing reports for functional safety purposes that summarize the findings and highlight any issues during analysis that could compromise the integrity of those findings. +- *Deviations* - a process for suppressing valid results, and maintaining metadata The requirements for these additional components are taken from the [MISRA Compliance 2020](https://www.misra.org.uk/app/uploads/2021/06/MISRA-Compliance-2020.pdf) document. Further details of these use cases can be found in the [user manual](user_manual.md). @@ -71,30 +73,34 @@ The requirements for these additional components are taken from the [MISRA Compl ### Overview - * For each selected rule we will write one or more CodeQL queries that implement the rule (see section _Splitting a rule into multiple queries_). - * Queries will be grouped into CodeQL packs, according to the coding standard the rule comes from. - * To ensure consistency and increase the speed of development, we generate outline query files from the `rules.csv` specification file. - * Where a rule is duplicated across different standards, we will still create separate queries for each standard, but the implementation may be shared between the standards. This allows each version to provide different metadata, and to be enabled/disabled individually. - +- For each selected rule we will write one or more CodeQL queries that implement the rule (see section *Splitting a rule into multiple queries*). +- Queries will be grouped into CodeQL packs, according to the coding standard the rule comes from. +- To ensure consistency and increase the speed of development, we generate outline query files from the `rules.csv` specification file. +- Where a rule is duplicated across different standards, we will still create separate queries for each standard, but the implementation may be shared between the standards. This allows each version to provide different metadata, and to be enabled/disabled individually. + ### Architecture For each supported coding standard we will provide: + 1. A CodeQL query pack containing the queries that implement the designated rules. 2. A CodeQL query pack containing the unit tests ("qltests") for each of the queries. These packs will be organized by supported language. The current supported languages are: + - C++14 standardized by [ISO/IEC 14882:2014](https://www.iso.org/standard/64029.html) located in the directory `cpp`. - [C99] standardized by [ISO/IEC 9899:1999](https://www.iso.org/standard/29237.html) and C11 standardized by [ISO/IEC 9899:2011](https://www.iso.org/standard/57853.html). All are located in the directory `c`. For each language, we will also include: + 1. A CodeQL query pack containing "common" libraries, which provide support. 2. A CodeQL query pack containing tests for the "common" libraries. The standards packs will depend on the "common" pack for the given language. This will allow the different standards to share implementation libraries. In the repository, this will be organized as follows: -``` + +```text / / src/ @@ -141,9 +147,10 @@ The decision to split a rule into multiple queries should be driven by the follo In order to speed up rule development and ensure implementation consistency we have created a series of scripts that generate templated rule files based on the `rules.csv` rule specification file. This generation process works on a per-rule package basis, and is driven by the creation of a "rule package description file", describing the mapping from rules to queries which will implement those rules. For this, there is a three step process: - 1. Generate a rule package description file for a given rule package. - 2. Review each entry in the rule package description file, updating the names and properties of the queries that will be written to implement these rules. - 3. Generate rule files from the rule package description file for a given rule package. + +1. Generate a rule package description file for a given rule package. +2. Review each entry in the rule package description file, updating the names and properties of the queries that will be written to implement these rules. +3. Generate rule files from the rule package description file for a given rule package. After these scripts have been run each query specified in the rule package description file will have: @@ -176,7 +183,7 @@ pip install -r scripts/requirements.txt To generate the rule package description file, run the following script from the root of the repository: -``` +```bash python3.9 scripts/generate_rules/generate_package_description.py ``` @@ -185,23 +192,24 @@ This will produce a `.json` file in the `rule_packages` directory with the name #### Step 2: Review and update the rule package description file The rule package description file produced in previous step is a `json` file which has the following structure: - - * A rule package object, with properties for each coding standard. - * A coding standard object, with properties for each implemented rule. - * A rule object, with: - * A `properties` property specifying some key-value pairs describing properties of the rule. - * A `title`s property specifying the rule title (also known as the rule "headline"). - * A `queries` property, specifying an array of query objects - * A query object, with: - * A `description` property, which will be used to populate the `@description` query metadata property value for this query. - * A `kind` property, which will be used to populate the `@kind` query metadata property value for this query. - * A `name` property, which will be used to populate the `@name` query metadata property value for this query. - * A `precision` property, which will be used to populate the `@precision` query metadata property value for this query. - * A `severity` property, which will be used to populate the `@severity` query metadata property value for this query. - * A `short_name` property, which will be used in the filenames for each file generated for this query, most notable as the name of the generated `.ql` query file, as well as the query id. - * A `tags` property, which will be used to populate the `@tags` query metadata property value for this query. + +- A rule package object, with properties for each coding standard. +- A coding standard object, with properties for each implemented rule. +- A rule object, with: + - A `properties` property specifying some key-value pairs describing properties of the rule. + - A `title`s property specifying the rule title (also known as the rule "headline"). + - A `queries` property, specifying an array of query objects +- A query object, with: + - A `description` property, which will be used to populate the `@description` query metadata property value for this query. + - A `kind` property, which will be used to populate the `@kind` query metadata property value for this query. + - A `name` property, which will be used to populate the `@name` query metadata property value for this query. + - A `precision` property, which will be used to populate the `@precision` query metadata property value for this query. + - A `severity` property, which will be used to populate the `@severity` query metadata property value for this query. + - A `short_name` property, which will be used in the filenames for each file generated for this query, most notable as the name of the generated `.ql` query file, as well as the query id. + - A `tags` property, which will be used to populate the `@tags` query metadata property value for this query. For example, this is the first part of the `Exceptions2.json` package file: + ```json { "AUTOSAR": { @@ -236,70 +244,74 @@ The query metadata instructs the CodeQL how to handle the query and display its The `generate_package_description.py` script provides a "best-effort" approach to setting each of the properties. For that reason, the rule package description file must be reviewed and updated. For each rule: - - Review the rule text in the relevant standard, and determine the number of queries - - For each `query` object review and update the following properties: - - `description` - **_must not be empty and end with a full stop_** - will be blank, unless the rule headline was too long to fit in the `name` property, in which case it will contain the rule headline. If the `description` is blank, fill it in explaining _why_ this could be a problem by explaining the consequences (see the CodeQL [metadata descriptions](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md#query-descriptions-description) documentation for more details). - - `kind` - pre-populated to `problem`. Modify to `path-problem` if this query is likely to use path explanations - for example, to explain data flow path. - - `name` - will be pre-populated the first 100 characters of the rule headline text, truncated at a sensible point. This should be a single sentence, and **_must not end in a full stop_**. - - `precision` - pre-populated based on a "difficulty" column present in the `rules.csv`. Set according to the definition specified in the CodeQL [metadata properties](https://codeql.github.com/docs/writing-codeql-queries/metadata-for-codeql-queries/#metadata-properties) documentation. - - `severity` - will be pre-populated to `error`, but should be adjusted based on the query. The criteria is that if the query does report a true positive - - `error` - if the reported issue is either directly a security vulnerability, or directly causes a bug or crash in the program. - - `warning` - if the reported issue is not an error, but could indirectly lead to a security vulnerability or a bug or crash in the program. - - `recommendation` - if the reported issue is primarily a stylistic or maintainability issue. - - `short_name` - must be a PascalCase string without spaces, which will be used for the name of the query file and to generate a query id. Pre-populated heuristically from from the rule headline text. Make adjustments as appropriate: - - The short name must not exceed 50 characters. - - Consider whether the query can be described more succinctly. For example `OnlyInstancesOfTypesDerivedFromExceptionShouldBeThrown` can be summarized more clearly as `OnlyThrowStdExceptionDerivedTypes`. - - `tags` - Apply at least one tag from the possible values listed below. If you want to use a query that is not listed a new tag can be added through a PR that modifies the possible tag values in the `query` sub-schema located in `schemas/rule-package.schema.json` and updates the list of possible values described below. - - `correctness` - if the query identifies incorrect program behavior. - - `security` - if the query identifies a potential security vulnerability. - - `readability` - if the query identifies an issue which makes the code harder to read. - - `maintainability` - if the query identifies an issue which makes the code harder to maintain. - - `performance` - if the query identifies an issue which has a negative impact on the performance of the code. - - `concurrency` - if the query identifies a concurrency issue. - - Validate the rule package description file using the `validate-rule-package.py` script that validates the rule package descriptions against the schema `rule-package.schema.json` located in the `schemas` directory. - - `python3 scripts/validate-rule-package.py ` - -#### Step 3: +- Review the rule text in the relevant standard, and determine the number of queries +- For each `query` object review and update the following properties: + - `description` - ***must not be empty and end with a full stop*** - will be blank, unless the rule headline was too long to fit in the `name` property, in which case it will contain the rule headline. If the `description` is blank, fill it in explaining *why* this could be a problem by explaining the consequences (see the CodeQL [metadata descriptions](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md#query-descriptions-description) documentation for more details). + - `kind` - pre-populated to `problem`. Modify to `path-problem` if this query is likely to use path explanations - for example, to explain data flow path. + - `name` - will be pre-populated the first 100 characters of the rule headline text, truncated at a sensible point. This should be a single sentence, and ***must not end in a full stop***. + - `precision` - pre-populated based on a "difficulty" column present in the `rules.csv`. Set according to the definition specified in the CodeQL [metadata properties](https://codeql.github.com/docs/writing-codeql-queries/metadata-for-codeql-queries/#metadata-properties) documentation. + - `severity` - will be pre-populated to `error`, but should be adjusted based on the query. The criteria is that if the query does report a true positive + - `error` - if the reported issue is either directly a security vulnerability, or directly causes a bug or crash in the program. + - `warning` - if the reported issue is not an error, but could indirectly lead to a security vulnerability or a bug or crash in the program. + - `recommendation` - if the reported issue is primarily a stylistic or maintainability issue. + - `short_name` - must be a PascalCase string without spaces, which will be used for the name of the query file and to generate a query id. Pre-populated heuristically from from the rule headline text. Make adjustments as appropriate: + - The short name must not exceed 50 characters. + - Consider whether the query can be described more succinctly. For example `OnlyInstancesOfTypesDerivedFromExceptionShouldBeThrown` can be summarized more clearly as `OnlyThrowStdExceptionDerivedTypes`. + - `tags` - Apply at least one tag from the possible values listed below. If you want to use a query that is not listed a new tag can be added through a PR that modifies the possible tag values in the `query` sub-schema located in `schemas/rule-package.schema.json` and updates the list of possible values described below. + - `correctness` - if the query identifies incorrect program behavior. + - `security` - if the query identifies a potential security vulnerability. + - `readability` - if the query identifies an issue which makes the code harder to read. + - `maintainability` - if the query identifies an issue which makes the code harder to maintain. + - `performance` - if the query identifies an issue which has a negative impact on the performance of the code. + - `concurrency` - if the query identifies a concurrency issue. + - Validate the rule package description file using the `validate-rule-package.py` script that validates the rule package descriptions against the schema `rule-package.schema.json` located in the `schemas` directory. + - `python3 scripts/validate-rule-package.py ` + +#### Step 3 Ensure that the repository [codeql-coding-standards-help](https://github.com/github/codeql-coding-standards-help) cloned as a sibling of the [codeql-coding-standards](https://github.com/github/codeql-coding-standards) repository switched to a branch that matches the branch your are working on. To generate the rule package files, run the following script from the root of the repository: -``` +```bash python3.9 scripts/generate_rules/generate_package_files.py ``` If the repository [codeql-coding-standards-help](https://github.com/github/codeql-coding-standards-help) is not cloned as a sibling, then run the script as follows: -``` +```bash python3.9 scripts/generate_rules/generate_package_files.py --external-help-dir ``` After running this script, the following files will be generated in the `//src/rules//` directory: - - A `.ql` query file with the query metadata pre-populated, and the standard imports included. - - A `.md` query help file with some boilerplate text describing the purpose of the query. + +- A `.ql` query file with the query metadata pre-populated, and the standard imports included. +- A `.md` query help file with some boilerplate text describing the purpose of the query. For the standards AUTOSAR and MISRA the help files will generated in the `//src/rules/` directory of the cloned [codeql-coding-standards-help](https://github.com/github/codeql-coding-standards-help) repository if available, otherwise the help file generation is skipped. In addition, the following files will be generated in the `//test/rules//` directory: - - An empty `test.cpp` or `test.c` file. - - A `.qlref` file, which refers to the generated query file. - - A `.expected` file, which contains some boiler plate text. This ensures that when qltest is run, it will not succeed, but it will allow the "Compare results" option in the CodeQL VS Code extension (which is only usually available with an `.expected` results file). + +- An empty `test.cpp` or `test.c` file. +- A `.qlref` file, which refers to the generated query file. +- A `.expected` file, which contains some boiler plate text. This ensures that when qltest is run, it will not succeed, but it will allow the "Compare results" option in the CodeQL VS Code extension (which is only usually available with an `.expected` results file). The script can be safely re-run, except in a few notable cases listed below. Re-running the script has the following effect: - - Overwrites`.qlref` file. - - Updates the autogenerated sections of the `.md` file. - - Touches the `test.cpp`, `test.c`, and `.expected` files, to ensure they exist on disk, but does not modify them if they exist. - - Updates the `.ql` query by overwriting the query metadata block only. The QL portion of the file is left untouched. + +- Overwrites`.qlref` file. +- Updates the autogenerated sections of the `.md` file. +- Touches the `test.cpp`, `test.c`, and `.expected` files, to ensure they exist on disk, but does not modify them if they exist. +- Updates the `.ql` query by overwriting the query metadata block only. The QL portion of the file is left untouched. The notable exceptions are: - - If a `query` object is deleted from the rule package description file, it will not be deleted on disk. - - If a `query` object has the `short_name` property modified in the rule package description file, query files will be created under the new name, but the query files for the old name will not be deleted. + +- If a `query` object is deleted from the rule package description file, it will not be deleted on disk. +- If a `query` object has the `short_name` property modified in the rule package description file, query files will be created under the new name, but the query files for the old name will not be deleted. ### Updating the query from the rule specification Updates to the rule specification require an update of the generated queries files. -As described in _step 3_ of the section _Generation of query templates from rule specifications_ the script `scripts/generate_rules/generate_package_files.py` can be safely re-run with the documented exceptions. +As described in *step 3* of the section *Generation of query templates from rule specifications* the script `scripts/generate_rules/generate_package_files.py` can be safely re-run with the documented exceptions. Each property of a query in the rule specification can be changed and the generated query files can be updated by rerunning the script `scripts/generate_rules/generate_package_files.py` with exception of the property `query.shortname`. Updating the `query.shortname` property is discussed in the next section. @@ -308,30 +320,31 @@ Each property of a query in the rule specification can be changed and the genera Changing the `query.shortname` property requires a manual update process with the following steps. 1. Determine the query who's `query.shortname` property needs to be updated. -2. Change the `query.shortname` value and generate the query files as described in _step 3_ of the section _Generation of query templates from rule specifications_. +2. Change the `query.shortname` value and generate the query files as described in *step 3* of the section *Generation of query templates from rule specifications*. 3. Migrate the query definition (excluding the query meta-data) from the previous query file to the new query file identified with the updated shortname. 4. Migrate the relevant sections from query help file from the previous query help file to the new help query file identified with the updated shortname. 5. Migrate the test case expected file identified by old `.expected` to the update `.expected` name. -6. Validate that the new test case passes by following the procedure described in the section _Running unit tests_. +6. Validate that the new test case passes by following the procedure described in the section *Running unit tests*. 7. Remove the following files with `git rm ` where `query.shortname` reflects the old shortname in the directory `//src/rules//`: - `.ql` - `.md` ### Query style guide -The following list describes the required style guides for a query that **must** be validated during the code-review process described in section _Code review and automated checks_. +The following list describes the required style guides for a query that **must** be validated during the code-review process described in section *Code review and automated checks*. A query **must** include: - - A use of the `isExcluded` predicate on the element reported as the primary location. This predicate ensures that we have a central mechanism for excluding results. This predicate may also be used on other elements relevant to the alert, but only if a suppression on that element should also cause alerts on the current element to be suppressed. - - A well formatted alert message: - - The message should be a complete standalone sentence, with punctuation and a full stop. - - The message should refer to this particular instance of the problem, rather than repeating the generic rule. e.g. "Call to banned function x." instead of "Do not use function x." - - Code elements should be placed in 'single quotes', unless they are formatted as links. - - Avoid value judgments such as "dubious" and "suspicious", and focus on factual statements about the problem. - - If possible, avoid constant alert messages. Either add placeholders and links (using $@), or concatenate element names to the alert message. Non-constant messages make it easier to find particular results, and links to other program elements can help provide additional context to help a developer understand the results. Examples: - - Instead of `Call to banned function.` prefer `Call to banned function foobar.`. - - Instead of `Return value from call is unused.` prefer `Return value from call to function [x] is unused.`, where `[x]` is a link to the function itself. - - Do not try to explain the solution in the message; instead that should be provided in the help for the query. + +- A use of the `isExcluded` predicate on the element reported as the primary location. This predicate ensures that we have a central mechanism for excluding results. This predicate may also be used on other elements relevant to the alert, but only if a suppression on that element should also cause alerts on the current element to be suppressed. +- A well formatted alert message: + - The message should be a complete standalone sentence, with punctuation and a full stop. + - The message should refer to this particular instance of the problem, rather than repeating the generic rule. e.g. "Call to banned function x." instead of "Do not use function x." + - Code elements should be placed in 'single quotes', unless they are formatted as links. + - Avoid value judgments such as "dubious" and "suspicious", and focus on factual statements about the problem. + - If possible, avoid constant alert messages. Either add placeholders and links (using $@), or concatenate element names to the alert message. Non-constant messages make it easier to find particular results, and links to other program elements can help provide additional context to help a developer understand the results. Examples: + - Instead of `Call to banned function.` prefer `Call to banned function foobar.`. + - Instead of `Return value from call is unused.` prefer `Return value from call to function [x] is unused.`, where `[x]` is a link to the function itself. + - Do not try to explain the solution in the message; instead that should be provided in the help for the query. All public predicates, classes, modules and files should be documented with QLDoc. All QLDoc should follow the [QLDoc style guide](https://github.com/github/codeql/blob/main/docs/qldoc-style-guide.md). @@ -344,21 +357,23 @@ Because the downloaded packs are cached, it is only necessary to run `install-pa ### Unit testing Every query which implements a rule **must** include: -- One or more unit tests. -- One or more unit tests for every non-trivial library. -- For each unit test both "compliant" and "non-compliant" test cases, and should exercise each different logical condition uniquely provided in the query, where possible within the testing framework. The scope of each test should be those conditions specific to this query. In particular, functionality provided by the CodeQL Standard Library for C++ does not need to be tested. + +- One or more unit tests. +- One or more unit tests for every non-trivial library. +- For each unit test both "compliant" and "non-compliant" test cases, and should exercise each different logical condition uniquely provided in the query, where possible within the testing framework. The scope of each test should be those conditions specific to this query. In particular, functionality provided by the CodeQL Standard Library for C++ does not need to be tested. #### Running unit tests During query development in VS Code, the unit tests can be run using the [testing features](https://codeql.github.com/docs/codeql-for-visual-studio-code/testing-codeql-queries-in-visual-studio-code/) in the CodeQL extension. Unit tests can also be run on the command line using the CodeQL CLI. With an appropriate CodeQL CLI (as specified in the `supported_codeql_configs.json` at the root of the repository), you can run the following from the root of the repository: -``` + +```bash codeql test run --show-extractor-output path/to/test/directory ``` -* `--show-extractor-output` - this shows the output from the extractor. It is most useful when the test fails because the file is not valid C++, where the extractor output will include the compilation failure. This is not shown in VS Code. -* `path/to/test/directory` - this can be a qlref file (like `cpp/autosar/test/rules/A15-2-2/`), a rule directory (`cpp/autosar/test/rules/A15-2-2/`) or a test qlpack (`cpp/autosar/test/`). +- `--show-extractor-output` - this shows the output from the extractor. It is most useful when the test fails because the file is not valid C++, where the extractor output will include the compilation failure. This is not shown in VS Code. +- `path/to/test/directory` - this can be a qlref file (like `cpp/autosar/test/rules/A15-2-2/`), a rule directory (`cpp/autosar/test/rules/A15-2-2/`) or a test qlpack (`cpp/autosar/test/`). For more details on running unit tests with the CodeQL CLI see the [Testing custom queries](https://codeql.github.com/docs/codeql-cli/testing-custom-queries/) help topic. @@ -366,27 +381,31 @@ For more details on running unit tests with the CodeQL CLI see the [Testing cust The C++ test cases **must** be formatted with `clang_format`. - - Test functions should be called `test_`, where `` is a brief description of this test case. +- Test functions should be called `test_`, where `` is a brief description of this test case. If possible, use meaningful names for elements in test cases. Where arbitrary names are required, you may use the following: - - Local variables should be named `l`, with i incremented for each new variable. - - Global variables should be named `g`, with i incremented for each new variable. - - Functions should be named `f`, with i incremented for each new variable. - - Member variables should be named `m`, with i incremented for each new variable. +- Local variables should be named `l`, with i incremented for each new variable. +- Global variables should be named `g`, with i incremented for each new variable. +- Functions should be named `f`, with i incremented for each new variable. +- Member variables should be named `m`, with i incremented for each new variable. Test cases **must** be annotated with a line-ending comment in this format: -``` + +```regexp (COMPLIANT(\[FALSE_POSITIVE\])?|NON_COMPLIANT(\[FALSE_NEGATIVE\])?)( - .*)? ``` + Where: - - `COMPLIANT` is added if the line represents a "compliant" test case - - The annotation `[FALSE_POSITIVE]` is added if the query currently reports this result. - - `NON_COMPLIANT` is chosen if the line represents a non-compliant test case - - The annotation `[FALSE_NEGATIVE]` is added if the query currently does not report this result. + +- `COMPLIANT` is added if the line represents a "compliant" test case + - The annotation `[FALSE_POSITIVE]` is added if the query currently reports this result. +- `NON_COMPLIANT` is chosen if the line represents a non-compliant test case + - The annotation `[FALSE_NEGATIVE]` is added if the query currently does not report this result. For example: -``` + +```cpp "\s"; // NON_COMPLIANT[FALSE_NEGATIVE] "\n"; // COMPLIANT "\U00000024"; // COMPLIANT[FALSE_POSITIVE] @@ -395,11 +414,12 @@ For example: #### Copying test code Like the `github/codeql` repository, the contents of our test files should not be copied from external sources (third-party code, personal projects, standard libraries). The only exceptions to this rule are the copying of declarations from: - - [ISO/IEC Programming languages - C](https://www.iso.org/standard/74528.html) (all versions) - - [ISO/IEC Programming languages - C++](https://www.iso.org/standard/68564.html) (all versions) - - Code from existing queries and tests in the `github/codeql` repository. - - Code from existing queries and tests in this repository. - - Code in the public domain + +- [ISO/IEC Programming languages - C](https://www.iso.org/standard/74528.html) (all versions) +- [ISO/IEC Programming languages - C++](https://www.iso.org/standard/68564.html) (all versions) +- Code from existing queries and tests in the `github/codeql` repository. +- Code from existing queries and tests in this repository. +- Code in the public domain This policy is based on the public policy for `github/codeql` as specified at [github/codeql: C++ Unit Tests - Copying code](https://github.com/github/codeql/blob/main/cpp/ql/test/README.md#copying-code). @@ -415,8 +435,8 @@ We have therefore implemented a partial "stub" standard library in the `cpp/comm Each proposed changed to `main` or a release branch is required to go through a code review process. This involves: - - A review and explicit approval by at least one other team member with "Write" access to the repository. - - Running automated checks that validate and verify the change and ensuring they pass. +- A review and explicit approval by at least one other team member with "Write" access to the repository. +- Running automated checks that validate and verify the change and ensuring they pass. This is implemented by requiring that proposed changes are submitted as pull requests to the GitHub repository hosting the queries, and is enforced by enabling GitHub [branch protection](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches) policies on the `main` and the release branches. @@ -426,10 +446,10 @@ An approving review and a "passing" state from every "Required" automated check The following automated checks are run on every push and pull request to `main` and to the release branches: - * Running the CodeQL Coding Standard unit tests against supported CodeQL CLIs and CodeQL Standard Libraries for C++. - * Validating that release artifacts can be created for that branch. - * Validating style rules for queries and test files. - * Confirming that the query help files are valid. +- Running the CodeQL Coding Standard unit tests against supported CodeQL CLIs and CodeQL Standard Libraries for C++. +- Validating that release artifacts can be created for that branch. +- Validating style rules for queries and test files. +- Confirming that the query help files are valid. These automated checks should pass before the pull request is merged. @@ -446,18 +466,22 @@ For proposed changes that modify the released artifacts an entry must be include For proposed changes which only add new queries or support for new rules, this process is fully automated, by reviewing differences in rule package metadata files between releases. For proposed changes which change: - - The structure or layout of the release artifacts. - - The evaluation performance (memory, execution time) of an existing query. - - The results of an existing query. -A _change note_ must be added to the `change_notes` directory. The format of the change notes is to create a file with a name matching the following pattern: -``` +- The structure or layout of the release artifacts. +- The evaluation performance (memory, execution time) of an existing query. +- The results of an existing query. + +A *change note* must be added to the `change_notes` directory. The format of the change notes is to create a file with a name matching the following pattern: + +```bash YYYY-MM-DD-short-name-for-issue.md ``` + For example `2021-06-29-remove-incompatibility-codeql-cli-2.5.6.md`. The contents of the file should be a markdown list (using `-`) with a user facing message specifying the nature of the change. If the changes relate to specific queries, then the top-level entry should specify the rule and query, and should provide a nested list of the changes. For example: -``` + +```md - `A12-8-6` - `CopyAndMoveNotDeclaredProtected.ql`: - Fixed issue #174 - a result is now only reported when the declaring class is either used as a base class in the database, or where the class is abstract. - Fixed a bug where exclusions did not apply to invalid assignment operators. @@ -468,61 +492,84 @@ The contents of the file should be a markdown list (using `-`) with a user facin ### External dependencies There are two external dependencies required for running the coding standards queries: - 1. The CodeQL CLI, the command line tool for building CodeQL databases and running queries over those databases. - 2. The CodeQL Standard Library + +1. The CodeQL CLI, the command line tool for building CodeQL databases and running queries over those databases. +2. The CodeQL Standard Library For the purpose of this repository, and any tool qualification, we consider these external dependencies to be "black boxes" which require verification when upgrading. To (a) clearly specify the supported versions of these external dependencies and to (b) enable automation around them, the repository contains a `supported_codeql_configs.json` which lists the sets of supported configurations. There are four fields: - * `codeql_cli` - this is the plain version number of the supported CodeQL CLI, e.g. `2.6.3`. - * `codeql_standard_library` - this is the name of a tag on the `github.com/github/codeql` repository. The tag should be compatible with the CodeQL CLI given above. This would typically use the `codeql-cli/v` tag for the release, although any tag which is compatible is allowed. - * `codeql_cli_bundle` - (optional) - if present, describes the CodeQL CLI bundle version that is compatible. The bundle should include precisely the CodeQL CLI version and CodeQL Standard Library versions specified in the two mandatory fields. - * `ghes` - (optional) - if present describes the GitHub Enterprise Server release whose integrated copy of the CodeQL Action points to the CodeQL CLI bundle specified in the `codeql_cli_bundle` field. +- `codeql_cli` - this is the plain version number of the supported CodeQL CLI, e.g. `2.6.3`. +- `codeql_standard_library` - this is the name of a tag on the `github.com/github/codeql` repository. The tag should be compatible with the CodeQL CLI given above. This would typically use the `codeql-cli/v` tag for the release, although any tag which is compatible is allowed. +- `codeql_cli_bundle` - (optional) - if present, describes the CodeQL CLI bundle version that is compatible. The bundle should include precisely the CodeQL CLI version and CodeQL Standard Library versions specified in the two mandatory fields. +- `ghes` - (optional) - if present describes the GitHub Enterprise Server release whose integrated copy of the CodeQL Action points to the CodeQL CLI bundle specified in the `codeql_cli_bundle` field. #### Upgrading external dependencies To upgrade the CodeQL external dependencies: - 1. Determine appropriate versions of the CodeQL CLI and `github/codeql` repository, according to the release schedule and customer demands. - 2. Determine if there is a compatible CodeQL CLI bundle version by looking at the releases specified at https://github.com/github/codeql-action/releases. The bundle always includes the standard library at the version specified by the `codeql-cli/v` tag in the `github/codeql` repository. - 3. If you find a compatible CodeQL CLI bundle, determine whether that bundle was released in a GitHub Enterprise server release, by inspecting the `defaults.json` file at https://github.com/github/codeql-action/blob/main/lib/defaults.json#L2 for the CodeQL Action submitted with - 4. Populated the `supported_codeql_configs.json` file with the given values, ensuring to delete the optional fields if they are not populated. - 5. Update the `codeql_modules/codeql` submodule pointer to the `codeql_standard_library` tag identified. - 6. Submit a Pull Request to the `github/codeql-coding-standards` repository with the title `Upgrade `github/codeql` dependency to `. Use this template for the description, filling : - ``` - This PR updates the `supported_codeql_configs.json` file to target: - - - CodeQL CLI - - CodeQL Standard Library - - GHES - - CodeQL CLI Bundle - - > - - - ## CodeQL dependency upgrade checklist: - - - [ ] Reformat our CodeQL using the latest version (if required) - - [ ] Identify any CodeQL compiler warnings and errors, and update queries as required. - - [ ] Validate that the `github/codeql` test cases succeed. - - [ ] Address any CodeQL test failures in the `github/codeql-coding-standards` repository. - - [ ] Validate performance vs pre-upgrade - ``` - 7. Follow the dependency upgrade checklist, confirming each step. The `.github/workflows/standard_library_upgrade_tests.yml` will trigger automation for running the `github/codeql` unit tests with the appropriate CLI version. - 8. Once all the automate tests have passed, and the checklist is complete, the PR can be merged. - 9. An internal notification should be shared with the development team. +1. Determine appropriate versions of the CodeQL CLI and `github/codeql` repository, according to the release schedule and customer demands. +2. Determine if there is a compatible CodeQL CLI bundle version by looking at the releases specified at [CodeQL Action releases](https://github.com/github/codeql-action/releases). The bundle always includes the standard library at the version specified by the `codeql-cli/v` tag in the `github/codeql` repository. +3. If you find a compatible CodeQL CLI bundle, determine whether that bundle was released in a GitHub Enterprise server release, by inspecting the `defaults.json` file at https://github.com/github/codeql-action/blob/main/lib/defaults.json#L2 for the CodeQL Action submitted with +4. Populated the `supported_codeql_configs.json` file with the given values, ensuring to delete the optional fields if they are not populated. +5. Update the `codeql_modules/codeql` submodule pointer to the `codeql_standard_library` tag identified. +6. Submit a Pull Request to the `github/codeql-coding-standards` repository with the title `Upgrade `github/codeql` dependency to `. Use this template for the description, filling : + + ```md + This PR updates the `supported_codeql_configs.json` file to target: + + - CodeQL CLI + - CodeQL Standard Library + - GHES + - CodeQL CLI Bundle + + > + + + ## CodeQL dependency upgrade checklist: + + - [ ] Reformat our CodeQL using the latest version (if required) + - [ ] Identify any CodeQL compiler warnings and errors, and update queries as required. + - [ ] Validate that the `github/codeql` test cases succeed. + - [ ] Address any CodeQL test failures in the `github/codeql-coding-standards` repository. + - [ ] Validate performance vs pre-upgrade + ``` + +7. Follow the dependency upgrade checklist, confirming each step. The `.github/workflows/standard_library_upgrade_tests.yml` will trigger automation for running the `github/codeql` unit tests with the appropriate CLI version. +8. Once all the automate tests have passed, and the checklist is complete, the PR can be merged. +9. An internal notification should be shared with the development team. ### Release process -#### Version Numbering +The release process is a combination of release specific Action workflows and validation Action workflows executed on each PR. +The flowchart below provides an overview of the release process and how the release specific Action workflows are related. + +```mermaid +flowchart TD; + prepare-release["Prepare release (prepare-release.yml)"] + validate-release["Validate release (validate-release.yml)"] + compiler-validation["Compiler tests (release-engineering/release-compiler-validation.yml.)"] + performance-testing["Performance testing (release-engineering/release-performance-testing.yml)"] + existing-checks["Existing checks run on each PR"] + update-release["Update release (update-release.yml)"] + finalize-release["Finalize release (finalize-release.yml)"] + + prepare-release-->validate-release + validate-release-->compiler-validation-->update-release + validate-release-->performance-testing-->update-release + prepare-release-->existing-checks-->update-release + update-release-->finalize-release +``` + +#### Version Numbering -Version numbers follow semantic versioning and adhere to the following guidelines specific to Coding Standards. +Version numbers follow semantic versioning and adhere to the following guidelines specific to Coding Standards. Given the version `..`: 1. If the release only fixes bugs, increment the `PATCH` number only. -2. If a release contains additional queries, increment the `MINOR` version number and set the `PATCH` number to 0. Note this may also contain fixes in addition to new queries. +2. If a release contains additional queries, increment the `MINOR` version number and set the `PATCH` number to 0. Note this may also contain fixes in addition to new queries. 3. Otherwise, if the release contains breaking changes such as removing queries, increment the `MAJOR` version number and set `MINOR` and `PATCH` to zero. #### Release management @@ -531,63 +578,58 @@ We use the "Releases" feature in GitHub to manage and track our releases. This p To simplify the process of generating the release information, the repository contains a number of scripts and Action workflows: - - [`generate_release_notes.py`](../scripts/release/generate_release_notes.py) - a script for generating release notes based on the contents of the repository in comparison to the previous release. - - [`create_draft_release.sh`](../scripts/release/create_draft_release.sh) - a script for creating a release by: - 1. Downloading the appropriate artifacts - 2. Generating the release notes by calling `generate_release_notes.py` with appropriate parameters - 3. Generating the list of supported rules - 4. Creating a draft release on GitHub containing the artifacts from the previous steps - 5. Triggering integration testing on the new release. - - [`create-draft-release.yml`](../.github/workflows/create-draft-release.yml) - a GitHub Actions workflow for running the `create_draft_release.sh` on demand within the CI/CD environment. +- [prepare-release.yml](./github/workflows/prepare-release.yml): The entry point for starting a new release. When provided with a version and a Git reference this workflow will + - Create a release branch. + - Create a release PR that will contain all the changes required for a release and will validate the release using checks. + - Create a draft release that will be updated during various stages of the release. +- [update-release.yml](./github/workflows/update-release.yml): This workflow will update the draft release when all checks have passed successfully on the release PR. The draft release is updated to: + - Have the most recent release notes as generated by the [update-release-notes.py](scripts/release/update-release-notes.py) script. + - Have the most recent release assets as generated by the [update-release-assets.py](scripts/release/update-release-assets.py). +- [finalize-release.yml](.github/workflows/finalize-release.yml): This will update the release tag and mark the release public when the release PR is merged to successfully conclude the release. +- [update-release-status.yml](.github/workflows/update-release-status.yml): This workflow will update the status on the release by monitoring the status of individual validation steps. When all succeeded this will invoke the `update-release.yml` workflow. +- [update-check-run.yml](.github/workflows/update-check-run.yml): Utility workflow that allow authorized external workflows (i.e., workflows in other repositories) to update the status of check runs in the coding standards repository. +- [validate-release.yml](.github/workflows/validate-release.yml): Utility workflow that will start the performance and compiler compatibility testing that are orchestrated from the codeql-coding-standards-release-engineering repository. #### Branching workflow -Each new major or minor release should have a dedicated release branch, with the name `rc/.`. A new patch version should re-use the existing release branch for the release that is being patched. +Each release should have a dedicated release branch, with the name `rc/..`. A new patch version should branch from the existing release branch for the release that is being patched. Ensure that the same release branch is created in the [codeql-coding-standards-help](https://github.com/github/codeql-coding-standards-help) repository. -#### Artifact creation +#### Release assets -There is an automated CI/CD job ([Code Scanning Query Pack Generation](../.github/workflows/code-scanning-pack-gen.yml)) provided that generates the following release artifacts for Coding Standards: +There is an automated CI/CD job ([Update Release](../.github/workflows/update-release.yml)) that will automatically generate the release assets according to the [release layout specification](scripts/release/release-layout.yml). +Among the assets are: - - Code Scanning query pack - generates a zipped folder that can be used with the CodeQL CLI directly, or with GitHub Advanced Security. +- Certification kit containing the proof obligations for ISO26262 certification. +- Code Scanning query packs that can be used with the CodeQL CLI directly, or with GitHub Advanced Security. **Use of Code Scanning within GitHub Advanced Security is not in scope for ISO 26262 tool qualification. See [user_manual.md#github-advanced-security](user_manual.md#github-advanced-security) for more information**. -These run on every push to `main` and `rc/*`, and on every pull request, and are releasable without modification, assuming all other status checks succeed on the same commit. - #### Creating a release +**NOTE**: If this is a hotfix release, make sure to invoke `prepare-release.yml` with `hotfix` set to `true`. + To create a new release: - 1. Create an internal "release checklist" issue. - 2. Determine the appropriate release version. Version numbers are generated + + 1. Determine the appropriate release version. Version numbers are generated according to the guidelines in the section "Version Numbering." - 3. If a new `MAJOR` version is necessary, create a new `rc/.0` branch off of `main`. Otherwise, reuse the existing `rc` branch and merge work from `main` into the `rc` branch you have selected. - 4. Ensure the same `rc` branch exists in the [codeql-coding-standards-help](https://github.com/github/codeql-coding-standards-help) repository. This branch will be used to include external help files. - 5. Submit a PR to update the `qlpack.yml` version numbers on the `main` branch to the next anticipated release. - 6. Submit a PR to update the `qlpack.yml` version numbers on the release branch to the new version. - 7. Trigger a [workflow dispatch event](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) for the [Create draft release](../.github/workflows/create-draft-release.yml) workflow, specifying the release branch. The output of this workflow should report a link to the draft release and a link to the integration testing workflow triggered for this release. - - In the event the workflow is unusable, the [`create_draft_release.sh`](../scripts/release/create_draft_release.sh) script can be run directly on a local machine. - 8. Run the following workflows with the new version number, e.g., `v2.0.0`: - - [Test Linux/x86_64](https://github.com/github/codeql-coding-standards-release-engineering/actions/workflows/test-release-performance-linux-x86_64.yml) - - [Test Windows/x86_64](https://github.com/github/codeql-coding-standards-release-engineering/actions/workflows/test-release-performance-windows-x86_64.yml) - - [Regenerate Performance Views](https://github.com/github/codeql-coding-standards-release-engineering/actions/workflows/regenerate-performance-views.yml) - 9. Confirm the integration testing workflow completes successfully, and that the execution time is comparable to previous releases, taking into account that the execution time is expected to increase proportionally as more queries are added for each release. Results may be viewed on the release engineering repo: https://github.com/github/codeql-coding-standards-release-engineering - 10. For release 1.0.0 and above, the integration testing results must be verified. For each "integration testing codebase": - - Download the SARIF result file - - Compare the results against the previously computed set of results for that integration testing codebase, and, for any new or changed results, spot check to confirm validity. - - For false positives and false negatives identified during this process issues should be opened on this repository to track the problems identified. - - For each issue opened, assess whether they are "significant" i.e. whether they are likely to cause problems in practice with customers. If so, consider Step 7. failed. - 11. If the release fails steps 7. or 8. (if applicable), retain the draft release, and rename it to `vminor.major.patch-rc`. Address the release blocking issues on the `rc/.` branch, and restart the release process at Step 7. - 12. If steps 7. and 8. (if applicable) succeeded, then the release can be marked as "published". - 13. Release artifacts can now be distributed to customers. - 14. Create an internal "release retrospective" issue, and document any pain points or other issues. - 15. Create a PR that merges the release candidate branch into `main`. + 2. Determine the appropriate [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) to base the new release on. For new major or minor releases, this will be `main`. For patch releases this will be the release branch that is patched. + 3. Trigger a [workflow dispatch event](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) for the [Prepare CodeQL Coding Standards release](../.github/workflows/prepare-release.yml) workflow, specifying the release version for the input `version` and the Git reference for the input `ref`, and `hotfix` with the value `true` **if** it is a hotfix release. + 4. Validate the compiler and performance results linked from their respective check runs in the PR's checks overview. + 1. Validate the performance results by ensuring the release performance doesn't regresses from the previous release by more than a factor of 2 without a good reason. + 2. Validate the compiler results by ensuring there is an acceptable number of compatibility issues. + 5. Merge the PR that is created for the release, named `Release v..` where ``, ``, and `` match with the input `version` of the workflow [Prepare CodeQL Coding Standards release](../.github/workflows/prepare-release.yml) triggered in the previous step. + 6. Merge the PRs for the performance and compiler validation results on the release engineering repository. + +The release automation consists of many test and validation steps that can fail. These can be addressed and the release can be restarted from step 3. +A restart of a release (i.e., calling `prepare-release.yml`) **WILL RECREATE THE EXISTING RELEASE BRANCH AND RELEASE PR**. Any additional changes added to the PR **MUST** be reapplied. +If a release has been marked public, the release can no longer be restarted or re-released without removing the release manually. ## False Positive Triage Rubric -When triaging issues in Coding Standards, please refer to the following rubric for making classifications. +When triaging issues in Coding Standards, please refer to the following rubric for making classifications. -**Impact** +### Impact | Level | Definition | | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -595,7 +637,7 @@ When triaging issues in Coding Standards, please refer to the following rubric f | Impact-Medium | Issue occurs in production code bases with relatively low to moderate frequency. Issue may or may not be considered disruptive to customer. | | Impact-Low | Issue may not occur in production code bases and may require hand crafted examples to surface. If the issue occurs in production code bases it occurs either infrequently or impacts only a few codebases. | -**Difficulty** +### Difficulty | Level | Definition | | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -616,20 +658,19 @@ Requirements and project planning are maintained separately within an internal r ### Purpose ot the `next` branch -This git repository also has a [`next` branch](https://github.com/github/codeql-coding-standards/tree/next). The purpose of this branch is to track changes that that will become necessary when upgrading the CodeQL external dependencies as described in section _Upgrading external dependencies_. The changes on the `next` branch will undergo only light reviewing. As such, a full review as described in section _Code review and automated checks_ is required when merging these changes into `main`; no releases should be made from the `next` branch. We aim to ensure that the changes on the `next` branch are as complete as possible so that merging into `main` will be straightforward. +This git repository also has a [`next` branch](https://github.com/github/codeql-coding-standards/tree/next). The purpose of this branch is to track changes that that will become necessary when upgrading the CodeQL external dependencies as described in section *Upgrading external dependencies*. The changes on the `next` branch will undergo only light reviewing. As such, a full review as described in section *Code review and automated checks* is required when merging these changes into `main`; no releases should be made from the `next` branch. We aim to ensure that the changes on the `next` branch are as complete as possible so that merging into `main` will be straightforward. ## Task Automation -In the `.vscode` directory this repository comes with a `tasks.json` file which automates some of the tasks described in this document. To access them, in VSCode use `Ctrl+Shift+P` and select `Run Task`. +In the `.vscode` directory this repository comes with a `tasks.json` file which automates some of the tasks described in this document. To access them, in VSCode use `Ctrl+Shift+P` and select `Run Task`. Available Tasks: 1. 🔥 Standards Automation: Initialize: Sets up your Python environment. -2. 📏 Standards Automation: Generate Rule Description File: Generates the rule description file for a package. -3. 📦 Standards Automation: Generate Package Files: Re/generates the files for a package. This command will remember your last arguments so you can just do `Rerun Last Task` in vscode unless you wish to change the arguments. +2. 📏 Standards Automation: Generate Rule Description File: Generates the rule description file for a package. +3. 📦 Standards Automation: Generate Package Files: Re/generates the files for a package. This command will remember your last arguments so you can just do `Rerun Last Task` in vscode unless you wish to change the arguments. 4. 📝 Standards Automation: Format CodeQL: Formats the current file with the codeql formatter. -5. ⚡ Standards Automation: Generated Expected Output: Generates the expected output from the current `.qlref` file in your `tests/` directory. - +5. ⚡ Standards Automation: Generated Expected Output: Generates the expected output from the current `.qlref` file in your `tests/` directory. ## Cookbook @@ -704,7 +745,7 @@ codeql test accept \ ### Troubleshooting: Unrecoverable mismatch between extractor and library dbschemes -The following error could be indicative of the Git submodule _codeql-coding-standards/github_modules_ being out-of-date: +The following error could be indicative of the Git submodule *codeql-coding-standards/github_modules* being out-of-date: >Could not upgrade the dataset in /path/to/codeql-coding-standards/cpp/autosar/test/rules/...: Unrecoverable mismatch between extractor and library dbschemes. diff --git a/scripts/bump_version.sh b/scripts/bump_version.sh deleted file mode 100644 index bc3e7495e3..0000000000 --- a/scripts/bump_version.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - - -if [[ -z $1 ]]; -then - echo "Usage: bump_version.sh " - exit -fi - -echo "Setting Local Branch Version to $1." - -# update the qlpacks -find . -name 'qlpack.yml' | grep -v './codeql_modules' | grep -v './scripts' | xargs sed -i "s/^version.*$/version: ${1}/" - -# update the documentation. - -find docs -name 'user_manual.md' | xargs sed -i "s/code-scanning-cpp-query-pack-.*\.zip\`/code-scanning-cpp-query-pack-${1}.zip\`/" -find docs -name 'user_manual.md' | xargs sed -i "s/supported_rules_list_.*\.csv\`/supported_rules_list_${1}.csv\`/" -find docs -name 'user_manual.md' | xargs sed -i "s/supported_rules_list_.*\.md\`/supported_rules_list_${1}.md\`/" -find docs -name 'user_manual.md' | xargs sed -i "s/user_manual_.*\.md\`/user_manual_${1}.md\`/" -find docs -name 'user_manual.md' | xargs sed -i "s/This user manual documents release \`.*\` of/This user manual documents release \`${1}\` of/" - -echo "Done." \ No newline at end of file diff --git a/scripts/release/bump-version.sh b/scripts/release/bump-version.sh new file mode 100755 index 0000000000..fd5ab5ea0d --- /dev/null +++ b/scripts/release/bump-version.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + + +if [[ -z $1 ]]; +then + echo "Usage: bump-version.sh " + exit +fi + +echo "Setting Local Branch Version to $1." + +# update the qlpacks +find . -name 'qlpack.yml' | grep -v './codeql_modules' | grep -v './scripts' | xargs sed -i "s/^version.*$/version: ${1}/" + +# update the documentation. + +find docs -name 'user_manual.md' -print0 | xargs -0 sed -i "s/code-scanning-cpp-query-pack-.*\.zip\`/code-scanning-cpp-query-pack-${1}.zip\`/" +find docs -name 'user_manual.md' -print0 | xargs -0 sed -i "s/supported_rules_list_.*\.csv\`/supported_rules_list_${1}.csv\`/" +find docs -name 'user_manual.md' -print0 | xargs -0 sed -i "s/supported_rules_list_.*\.md\`/supported_rules_list_${1}.md\`/" +find docs -name 'user_manual.md' -print0 | xargs -0 sed -i "s/user_manual_.*\.md\`/user_manual_${1}.md\`/" +find docs -name 'user_manual.md' -print0 | xargs -0 sed -i "s/This user manual documents release \`.*\` of/This user manual documents release \`${1}\` of/" + +echo "Done." \ No newline at end of file diff --git a/scripts/release/create_draft_release.sh b/scripts/release/create_draft_release.sh deleted file mode 100755 index fa3000d450..0000000000 --- a/scripts/release/create_draft_release.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Script for generating a draft release for the CodeQL Coding Standards repository, for the given branch. - -set -o errexit -set -o nounset - -BRANCH="$1" -VERSION="$2" - -if [[ ! $BRANCH == rc/* ]]; then - echo "$BRANCH is not an rc branch of the form rc/" - exit 1 -fi - -if [ -z "$VERSION" ]; then - VERSION="${BRANCH#rc/}" - echo "Version not set explicitly; auto-detecting $VERSION." -fi - -COMMIT_SHA="$(git rev-parse $BRANCH)" - -echo "Creating draft release for $VERSION from $BRANCH at commit $COMMIT_SHA." - -echo "Identifying code-scanning-pack-gen.yml" -CODE_SCANNING_PACK_GEN_RUN_ID=$(gh api -X GET repos/github/codeql-coding-standards/actions/workflows/code-scanning-pack-gen.yml/runs -F branch="$BRANCH" -F event="push" -F conclusion="success" --jq "first(.workflow_runs.[] | select(.head_sha==\"$COMMIT_SHA\") | .id)") -if [ -z "$CODE_SCANNING_PACK_GEN_RUN_ID" ]; then - echo "No successful run of the code-scanning-pack-gen.yml file for $COMMIT_SHA on branch $BRANCH." - exit 1 -fi - -# Create a temp directory to store the artifacts in -TEMP_DIR="$(mktemp -d)" - -echo "Identified code-scanning-pack-gen.yml run id: $CODE_SCANNING_PACK_GEN_RUN_ID" - -echo "Fetching Code Scanning pack" -CODE_SCANNING_ARTIFACT_NAME="code-scanning-cpp-query-pack.zip" -CODE_SCANNING_VERSIONED_ARTIFACT_NAME="code-scanning-cpp-query-pack-$VERSION.zip" -gh run download $CODE_SCANNING_PACK_GEN_RUN_ID -n "$CODE_SCANNING_ARTIFACT_NAME" -mv "$CODE_SCANNING_ARTIFACT_NAME" "$TEMP_DIR/$CODE_SCANNING_VERSIONED_ARTIFACT_NAME" - -echo "Generating release notes." -python3 scripts/release/generate_release_notes.py > "$TEMP_DIR/release_notes_$VERSION.md" -python3 scripts/release/create_supported_rules_list.py > "$TEMP_DIR/supported_rules_list_$VERSION.md" -python3 scripts/release/create_supported_rules_list.py --csv > "$TEMP_DIR/supported_rules_list_$VERSION.csv" - -echo "Copy Docs to Artifact Directory" -cp docs/user_manual.md "$TEMP_DIR/user_manual_$VERSION.md" - -echo "Generating Checksums" -sha256sum $TEMP_DIR/* > "$TEMP_DIR/checksums.txt" - -gh release create "v$VERSION" -d --target "$BRANCH" -F "$TEMP_DIR/release_notes_$VERSION.md" -t "v$VERSION" "$TEMP_DIR/$CODE_SCANNING_VERSIONED_ARTIFACT_NAME" "$TEMP_DIR/supported_rules_list_$VERSION.md" "$TEMP_DIR/checksums.txt" "$TEMP_DIR/supported_rules_list_$VERSION.csv" "$TEMP_DIR/user_manual_$VERSION.md" - -curl \ - -X POST \ - -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $INTEGRATION_TESTING_ACCESS_TOKEN" \ - https://api.github.com/repos/coding-standards-integration-testing/integration-testing-production/actions/workflows/$WORKFLOW_ID/dispatches \ - -d '{"ref":"refs/heads/main", "inputs": { "release_version_tag":"'"$VERSION"'", "codeql_analysis_threads":"'"$CODEQL_ANALYSIS_THREADS"'", "aws_ec2_instance_type":"'"$AWS_EC2_INSTANCE_TYPE"'" }}' diff --git a/scripts/release/create_supported_rules_list.py b/scripts/release/create_supported_rules_list.py index 15a8b5d6b7..e3294ed3b1 100644 --- a/scripts/release/create_supported_rules_list.py +++ b/scripts/release/create_supported_rules_list.py @@ -12,9 +12,6 @@ When run without any arguments, the script iterates through each of the rule package description files stored in the `rule_packages` directory, and identifies which rules are supported by one or more queries. - -This script needs to be run with the codeql-coding-standards git repository as the current working -directory. """ if (len(sys.argv) == 2 and sys.argv[1] == "--help"): diff --git a/scripts/release/is-hotfix-release.py b/scripts/release/is-hotfix-release.py new file mode 100644 index 0000000000..a496b63c27 --- /dev/null +++ b/scripts/release/is-hotfix-release.py @@ -0,0 +1,56 @@ +from semantic_version import Version # type: ignore +from subprocess import run +from typing import List, Literal, TYPE_CHECKING +from sys import stderr + +if TYPE_CHECKING: + from argparse import Namespace + +def get_merge_base_of_ref() -> str: + cp = run(["git", "merge-base", "HEAD", "origin/main"], capture_output=True, text=True) + if cp.returncode != 0: + raise RuntimeError("Failed to get merge base") + return cp.stdout.strip() + +def get_release_branches_containing(commit: str) -> List[Version]: + cp = run(["git", "branch", "--list", "rc/*", "--contains", commit], capture_output=True, text=True) + if cp.returncode != 0: + raise RuntimeError("Failed to get branches containing commit") + release_versions: List[Version] = [] + for version in [b.strip() for b in cp.stdout.splitlines()]: + try: + if version.startswith("rc/"): + version = version[3:] + release_versions.append(Version(version)) + except ValueError: + print(f"Warning: Skipping invalid version string: {version}", file=stderr) + + return release_versions + +def main(args: 'Namespace') -> Literal[0,1]: + try: + merge_base = get_merge_base_of_ref() + release_versions = get_release_branches_containing(merge_base) + if len(release_versions) == 0: + print(f"Info: No release branches found containing merge base {merge_base}", file=stderr) + print("false") + return 0 + + for version in release_versions: + if version.next_patch() == Version(args.version): + print("true") + return 0 + + print("false") + return 0 + except RuntimeError as e: + print(f"Error: {e}", file=stderr) + return 1 + +if __name__ == '__main__': + from sys import stderr, exit + import argparse + + parser = argparse.ArgumentParser(description="Check if a version is a hotfix release") + parser.add_argument("version", help="The version string to compare against the base branches") + exit(main(parser.parse_args())) \ No newline at end of file diff --git a/scripts/release/release-layout.yml b/scripts/release/release-layout.yml new file mode 100644 index 0000000000..3ffc3ba0de --- /dev/null +++ b/scripts/release/release-layout.yml @@ -0,0 +1,23 @@ +version: 0.1.0 + +layout: + certification_kit.zip: + - workflow-log: + name: ".*" + - workflow-artifact: + not-name: "Code Scanning Query Pack Generation" + code-scanning-cpp-query-pack.zip: + - workflow-artifact: + name: "Code Scanning Query Pack Generation" + artifact: code-scanning-cpp-query-pack.zip + supported_rules_list.csv: + - shell: | + python ${{ coding-standards.root }}/scripts/release/create_supported_rules_list.py --csv > supported_rules_list.csv + supported_rules_list.md: + - shell: | + python ${{ coding-standards.root }}/scripts/release/create_supported_rules_list.py > supported_rules_list.md + user_manual.md: + - file: docs/user_manual.md + checksums.txt: + - shell: | + sha256sum ./* > checksums.txt \ No newline at end of file diff --git a/scripts/release/requirements.txt b/scripts/release/requirements.txt new file mode 100644 index 0000000000..79ccbcefbe --- /dev/null +++ b/scripts/release/requirements.txt @@ -0,0 +1,4 @@ +semantic-version==2.10.0 +PyGithub==1.59.1 +PyYAML==6.0.1 +GitPython==3.1.36 \ No newline at end of file diff --git a/scripts/release/update-release-assets.py b/scripts/release/update-release-assets.py new file mode 100644 index 0000000000..79b06cbcfe --- /dev/null +++ b/scripts/release/update-release-assets.py @@ -0,0 +1,363 @@ +from __future__ import annotations # This enables postponed evaluation of type annotations. Required for typing.TYPE_CHECKING. See https://peps.python.org/pep-0563/ +from typing import TYPE_CHECKING, List, Union, cast, Dict, Any +import shutil +from tempfile import TemporaryDirectory +import subprocess +import re +from pathlib import Path +import sys +import semantic_version # type: ignore +import requests +import yaml + +if TYPE_CHECKING: + from github import WorkflowRun, Repository + + +script_path = Path(__file__).resolve() +root_path = script_path.parent.parent.parent + +def monkey_patch_github() -> None: + from github import Repository, PaginatedList, CheckRun + + class MyRepository(Repository.Repository): + def get_check_runs(self: Repository.Repository, ref: str, **kwargs: str) -> PaginatedList.PaginatedList[CheckRun.CheckRun]: + assert isinstance(ref, str), ref + + return PaginatedList.PaginatedList( + CheckRun.CheckRun, + self._requester, + f"{self.url}/commits/{ref}/check-runs", + firstParams=None, + list_item="check_runs") + + Repository.Repository = MyRepository + + from github import WorkflowRun, Artifact + class MyWorkflowRun(WorkflowRun.WorkflowRun): + def download_logs(self, path: Path) -> None: + """ + Download the logs for this workflow and store them in the directory specified by path. + + This method tries to minimize the dependency on the internal workings of the class Requester by using + requests directly. Ideally we would like to monkey patch __rawRequest to deal with 302 redirects, but + that didn't work out because it would fail to call other private methods with an AttributeError for an unknown reason. + """ + url = f"{self.url}/logs" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + if self._requester._Requester__auth is not None: # type: ignore + headers["Authorization"] = f"{self._requester._Requester__auth.token_type} {self._requester._Requester__auth.token}" # type: ignore + headers["User-Agent"] = self._requester._Requester__userAgent # type: ignore + + resp = requests.get(url, headers=headers, allow_redirects=True) + + if resp.status_code != 200: + raise Exception(f"Unable to download logs: {resp.status_code} {resp.reason}") + + with (path / f"{self.name}-{self.head_sha}-{self.run_number}.zip").open("wb") as f: + f.write(resp.content) + + def download_artifacts(self, path: Path) -> None: + for artifact in self.get_artifacts(): # type: ignore + artifact = cast(Artifact.Artifact, artifact) + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + if self._requester._Requester__auth is not None: # type: ignore + headers["Authorization"] = f"{self._requester._Requester__auth.token_type} {self._requester._Requester__auth.token}" # type: ignore + headers["User-Agent"] = self._requester._Requester__userAgent # type: ignore + + resp = requests.get(artifact.archive_download_url, headers=headers, allow_redirects=True) + + if resp.status_code != 200: + raise Exception(f"Unable to download artifact ${artifact.name}. Received status code {resp.status_code} {resp.reason}") + + with (path / f"{artifact.name}.zip").open("wb") as f: + f.write(resp.content) + + def download_artifact(self, name: str, path: Path) -> None: + candidates: List[Artifact.Artifact] = [artifact for artifact in self.get_artifacts() if artifact.name == name] # type: ignore + if len(candidates) == 0: + raise Exception(f"Unable to find artifact {name}") + assert(len(candidates) == 1) + + artifact = candidates[0] + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + } + if self._requester._Requester__auth is not None: # type: ignore + headers["Authorization"] = f"{self._requester._Requester__auth.token_type} {self._requester._Requester__auth.token}" # type: ignore + headers["User-Agent"] = self._requester._Requester__userAgent # type: ignore + + resp = requests.get(artifact.archive_download_url, headers=headers, allow_redirects=True) + + if resp.status_code != 200: + raise Exception(f"Unable to download artifact ${artifact.name}. Received status code {resp.status_code} {resp.reason}") + + with (path / f"{artifact.name}.zip").open("wb") as f: + f.write(resp.content) + + + WorkflowRun.WorkflowRun = MyWorkflowRun + +class ReleaseLayout: + def __init__(self, specification: Path, skip_checks: bool = False) -> None: + self.specification = specification + self.artifacts = [] + self.skip_checks = skip_checks + + def make(self, directory: Path, workflow_runs: List[WorkflowRun.WorkflowRun]) -> None: + spec = yaml.safe_load(self.specification.read_text()) + artifacts : List[ReleaseArtifact] = [] + for artifact, action_specs in spec["layout"].items(): + actions : List[Union[WorkflowArtifactAction, WorkflowLogAction, ShellAction, FileAction]] = [] + for action_spec in action_specs: + assert(len(action_spec) == 1) + action_type, action_args = action_spec.popitem() + if action_type == "workflow-log": + actions.append(WorkflowLogAction(workflow_runs, **cast(Dict[str, Any], action_args))) + elif action_type == "workflow-artifact": + actions.append(WorkflowArtifactAction(workflow_runs, **cast(Dict[str, Any], action_args))) + elif action_type == "shell": + actions.append(ShellAction(action_args)) + elif action_type == "file": + actions.append(FileAction(action_args)) + else: + raise Exception(f"Unknown action type {action_type}") + + artifacts.append(ReleaseArtifact(artifact, actions, self.skip_checks)) + + for artifact in artifacts: + artifact.make(directory) + +class WorkflowLogAction(): + + def __init__(self, workflow_runs: List[WorkflowRun.WorkflowRun], **kwargs: str) -> None: + self.workflow_runs = workflow_runs + self.kwargs: dict[str, str] = kwargs + self.temp_workdir = TemporaryDirectory() + + def run(self) -> List[Path]: + workflow_runs = self.workflow_runs + if "name" in self.kwargs: + workflow_runs = [workflow_run for workflow_run in self.workflow_runs if re.match(self.kwargs["name"], workflow_run.name)] + if "not-name" in self.kwargs: + workflow_runs = [workflow_run for workflow_run in self.workflow_runs if not re.match(self.kwargs["not-name"], workflow_run.name)] + print(f"Downloading the logs for {len(workflow_runs)} workflow runs") + for workflow_run in workflow_runs: + print(f"Downloading logs for {workflow_run.name}") + workflow_run.download_logs(Path(self.temp_workdir.name)) # type: ignore + return list(map(Path, Path(self.temp_workdir.name).glob("**/*"))) + +class WorkflowArtifactAction(): + + def __init__(self, workflow_runs: List[WorkflowRun.WorkflowRun], **kwargs: str) -> None: + self.workflow_runs = workflow_runs + self.kwargs: dict[str, str] = kwargs + self.temp_workdir = TemporaryDirectory() + + def run(self) -> List[Path]: + workflow_runs = self.workflow_runs + if "name" in self.kwargs: + workflow_runs = [workflow_run for workflow_run in self.workflow_runs if re.match(self.kwargs["name"], workflow_run.name)] + if "not-name" in self.kwargs: + workflow_runs = [workflow_run for workflow_run in self.workflow_runs if not re.match(self.kwargs["not-name"], workflow_run.name)] + print(f"Downloading the artifacts for {len(workflow_runs)} workflow runs") + for workflow_run in workflow_runs: + if "artifact" in self.kwargs: + print(f"Downloading artifact {self.kwargs['artifact']} for {workflow_run.name} to {self.temp_workdir.name}") + workflow_run.download_artifact(self.kwargs["artifact"], Path(self.temp_workdir.name)) # type: ignore + else: + print(f"Downloading artifacts for {workflow_run.name} to {self.temp_workdir.name}") + workflow_run.download_artifacts(Path(self.temp_workdir.name)) # type: ignore + return list(map(Path, Path(self.temp_workdir.name).glob("**/*"))) + +class ShellAction(): + def __init__(self, command: str) -> None: + self.command = command.strip() + self.temp_workdir = TemporaryDirectory() + + def run(self) -> List[Path]: + concrete_command = re.sub(pattern=r"\${{\s*coding-standards\.root\s*}}", repl=str(root_path), string=self.command) + subprocess.run(concrete_command, cwd=self.temp_workdir.name, check=True, shell=True) + return list(map(Path, Path(self.temp_workdir.name).glob("**/*"))) + +class FileAction(): + def __init__(self, path: Path) -> None: + self.path = path + + def run(self) -> List[Path]: + return [self.path] + +class ReleaseArtifact(): + def __init__(self, name: str, actions: List[Union[WorkflowLogAction, WorkflowArtifactAction, ShellAction, FileAction]], allow_no_files: bool = False) -> None: + self.name = Path(name) + self.actions = actions + self.allow_no_files = allow_no_files + + def make(self, directory: Path) -> Path: + files: list[Path] = [file for action in self.actions for file in action.run()] + if len(files) == 0: + if not self.allow_no_files: + raise Exception(f"Artifact {self.name} has no associated files!") + elif len(files) == 1: + shutil.copy(files[0], directory / self.name) + return directory / self.name + else: + extension = "".join(self.name.suffixes)[1:] + if not extension in ["zip", "tar", "tar.gz", "tar.bz2", "tar.xz"]: + raise Exception(f"Artifact {self.name} is not a support archive file, but has multiple files associated with it!") + + ext_format_map = { + "zip": "zip", + "tar": "tar", + "tar.gz": "gztar", + "tar.bz2": "bztar", + "tar.xz": "xztar" + } + + with TemporaryDirectory() as temp_dir: + temp_dir_path = Path(temp_dir) + for file in files: + shutil.copy(file, temp_dir_path / file.name) + + return Path(shutil.make_archive(str(directory / self.name.with_suffix("")), ext_format_map[extension], root_dir=temp_dir_path)) + +def main(args: 'argparse.Namespace') -> int: + monkey_patch_github() + + import github + from github import CheckRun + + repos: Dict[str, Repository.Repository] = {} + if len(args.github_token) == 1: + repos[args.repo] = github.Github(auth=github.Auth.Token(args.github_token[0])).get_repo(args.repo) + else: + for token in args.github_token: + nwo, token = token.split(":") + repos[nwo] = github.Github(auth=github.Auth.Token(token)).get_repo(nwo) + + repo = repos[args.repo] + + pull_candidates = [pr for pr in repo.get_pulls(state="open") if pr.head.sha == args.head_sha] + if len(pull_candidates) != 1: + print(f"Error: expected exactly one PR for SHA {args.head_sha}, but found {len(pull_candidates)}", file=sys.stderr) + return 1 + + pull_request = pull_candidates[0] + + if pull_request.state != "open": + print(f"Error: PR {pull_request.url} is not open", file=sys.stderr) + return 1 + + print(f"Found PR {pull_request.url} based on {pull_request.base.ref}") + + rc_branch_regex = r"^rc/(?P.*)$" + rc_branch_match = re.match(rc_branch_regex, pull_request.base.ref) + if not rc_branch_match: + print(f"Error: PR {pull_request.url} is not based on a release candidate branch", file=sys.stderr) + return 1 + + release_version = rc_branch_match.group("version") + + try: + semantic_version.Version.parse(release_version) # type: ignore + except ValueError as e: + print(f"Error: invalid version {release_version} use by release branch. Reason {e}", file=sys.stderr) + return 1 + + print(f"Looking for release with tag v{release_version} associated with the PR's base ref {pull_request.base.ref}") + all_releases = repo.get_releases() + for release in all_releases: + print(f"Found release {release.title} with tag {release.tag_name}") + releases = [release for release in all_releases if release.tag_name == f"v{release_version}"] + if len(releases) != 1: + print(f"Error: expected exactly one release for {release_version}, but found {len(releases)}", file=sys.stderr) + return 1 + release = releases[0] + + print(f"Collecting workflow runs for ref {args.head_sha}") + check_runs: List[CheckRun.CheckRun] = repo.get_check_runs(args.head_sha) # type: ignore + + action_workflow_run_url_regex = r"^https://(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/actions/runs/(?P\d+)$" + action_workflow_job_run_url_regex = r"^https://(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/actions/runs/(?P\d+)/job/(?P\d+)$" + + workflow_runs: List[WorkflowRun.WorkflowRun] = [] + for check_run in check_runs: # type: ignore + check_run = cast(CheckRun.CheckRun, check_run) + if check_run.name in args.skip_checkrun: + print(f"Skipping check run {check_run.name} with id {check_run.id} because it is on the skip list.") + continue + job_run_match = re.match(action_workflow_job_run_url_regex, check_run.details_url) + if job_run_match: + workflow_run = repo.get_workflow_run(int(job_run_match.group("run_id"))) + workflow_runs.append(workflow_run) + else: + run_match = re.match(action_workflow_run_url_regex, check_run.external_id) + if run_match: + #print(f"External workflow on {run_match.group('owner')} {run_match.group('repo')} with id {run_match.group('run_id')}") + workflow_run = repos[f"{run_match.group('owner')}/{run_match.group('repo')}"].get_workflow_run(int(run_match.group("run_id"))) + workflow_runs.append(workflow_run) + else: + print(f"Unable to handle checkrun {check_run.name} with id {check_run.id} with {check_run.details_url}") + return 1 + + print("Filtering workflow runs to only include the latest run for each workflow.") + workflow_runs_per_id: Dict[int, WorkflowRun.WorkflowRun] = {} + for workflow_run in workflow_runs: + if not workflow_run.id in workflow_runs_per_id: + workflow_runs_per_id[workflow_run.id] = workflow_run + else: + latest_run = workflow_runs_per_id[workflow_run.id] + if latest_run.run_number < workflow_run.run_number: + workflow_runs_per_id[workflow_run.id] = workflow_run + latests_workflow_runs = list(workflow_runs_per_id.values()) + + if not args.skip_checks: + print(f"Checking that all workflow runs for ref {args.head_sha} succeeded") + for workflow_run in latests_workflow_runs: + if workflow_run.status != "completed": + print(f"Error: workflow run {workflow_run.name} with id {workflow_run.id} is not completed", file=sys.stderr) + return 1 + # Consider success or skipped as success + if workflow_run.conclusion == "failure": + print(f"Error: workflow run {workflow_run.name} with id {workflow_run.id} failed", file=sys.stderr) + return 1 + + with TemporaryDirectory() as temp_dir: + print(f"Using temporary directory {temp_dir}") + try: + ReleaseLayout(Path(args.layout), args.skip_checks).make(Path(temp_dir), latests_workflow_runs) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + print("Deleting existing assets") + for asset in release.assets: + asset.delete_asset() + + print("Uploading new assets from generated release layout") + for file in Path(temp_dir).glob("**/*"): + print(f"Uploading {file}") + release.upload_asset(str(file)) + + return 0 + +if __name__ == '__main__': + import argparse + from sys import exit + + parser = argparse.ArgumentParser() + parser.add_argument('--head-sha', help="The head SHA of the release PR for which we update it's corresponding release", required=True) + parser.add_argument('--repo', help="The owner and repository name. For example, 'octocat/Hello-World'. Used when testing this script on a fork", required=True, default="github/codeql-coding-standards") + parser.add_argument('--github-token', help="The github token to access repo and the repositories provided as external ids in check runs. When multiple tokens are provided use the format 'owner/repo:token'", required=True, nargs="+") + parser.add_argument('--layout', help="The layout to use for the release", required=True) + parser.add_argument('--skip-checkrun', help="Name of check run to exclude from consideration. Can be specified multiple times", nargs='+', default=["release-status"]) + parser.add_argument('--skip-checks', help="Skip the checks that ensure that the workflow runs succeeded", action="store_true") + args = parser.parse_args() + exit(main(args)) \ No newline at end of file diff --git a/scripts/release/update-release-notes.py b/scripts/release/update-release-notes.py new file mode 100644 index 0000000000..5f317ad988 --- /dev/null +++ b/scripts/release/update-release-notes.py @@ -0,0 +1,72 @@ +from __future__ import annotations # This enables postponed evaluation of type annotations. Required for typing.TYPE_CHECKING. See https://peps.python.org/pep-0563/ +from typing import TYPE_CHECKING +import subprocess +from pathlib import Path + +if TYPE_CHECKING: + from argparse import Namespace + +def generate_release_notes() -> str: + script_path = Path(__file__).parent / "generate_release_notes.py" + cp = subprocess.run(["python", str(script_path)], capture_output=True) + + if cp.returncode != 0: + raise Exception(f"Error generating release notes: {cp.stderr.decode('utf-8')}") + + return cp.stdout.decode("utf-8") + +def main(args: Namespace) -> int: + from github import Github, Auth + import semantic_version # type: ignore + import re + import sys + + repo = Github(auth=Auth.Token(args.github_token)).get_repo(args.repo) + + pull_candidates = [pr for pr in repo.get_pulls(state="open") if pr.head.sha == args.head_sha] + if len(pull_candidates) != 1: + print(f"Error: expected exactly one PR for SHA {args.head_sha}, but found {len(pull_candidates)}", file=sys.stderr) + return 1 + + pull_request = pull_candidates[0] + + if pull_request.state != "open": + print(f"Error: PR for version {args.version} is not open", file=sys.stderr) + return 1 + + rc_branch_regex = r"^rc/(?P.*)$" + rc_branch_match = re.match(rc_branch_regex, pull_request.base.ref) + if not rc_branch_match: + print(f"Error: PR {pull_request.url} is not based on a release candidate branch", file=sys.stderr) + return 1 + + release_version = rc_branch_match.group("version") + + try: + semantic_version.Version.parse(release_version) # type: ignore + except ValueError as e: + print(f"Error: invalid version {release_version} use by release branch. Reason {e}", file=sys.stderr) + return 1 + + releases = [release for release in repo.get_releases() if release.title == f"v{release_version}"] + if len(releases) != 1: + print(f"Error: expected exactly one release with title {args.version}, but found {len(releases)}", file=sys.stderr) + return 1 + release = releases[0] + + release_notes = generate_release_notes() + + release.update_release(name=release.title, message=release_notes, draft=release.draft, prerelease=release.prerelease, tag_name=release.tag_name) + + return 0 + +if __name__ == '__main__': + import argparse + from sys import exit + + parser = argparse.ArgumentParser() + parser.add_argument('--head-sha', help="The head SHA of the release PR for which we update it's corresponding release", required=True) + parser.add_argument('--repo', help="The owner and repository name. For example, 'octocat/Hello-World'. Used when testing this script on a fork", required=True, default="github/codeql-coding-standards") + parser.add_argument('--github-token', help="The GitHub token to use to update the release", required=True) + args = parser.parse_args() + exit(main(args)) \ No newline at end of file diff --git a/scripts/release/utils.py b/scripts/release/utils.py index 4e9bb99dd2..cdb747c076 100644 --- a/scripts/release/utils.py +++ b/scripts/release/utils.py @@ -1,5 +1,6 @@ import re import yaml +from pathlib import Path def get_query_short_names(rule_dict): """Gets a list of the query "short_name" properties for the given rule""" @@ -18,7 +19,9 @@ def split_rule_id(rule_id): def get_standard_version(standard): """Gets the qlpack version for the given standard.""" - qlpack_path = "cpp/" + standard.split("-")[0].lower() + "/src/qlpack.yml" + module_path = Path(__file__) + repo_root = module_path.parent.parent.parent + qlpack_path = repo_root / "cpp" / standard.split("-")[0].lower() /"src" / "qlpack.yml" with open(qlpack_path, 'r') as qlpack_file: try: qlpack = yaml.safe_load(qlpack_file) diff --git a/scripts/release/validate-version.py b/scripts/release/validate-version.py new file mode 100644 index 0000000000..3e8168d5b1 --- /dev/null +++ b/scripts/release/validate-version.py @@ -0,0 +1,47 @@ +import semantic_version # type: ignore +from typing import Literal, TYPE_CHECKING +from subprocess import run + +if TYPE_CHECKING: + from argparse import Namespace + +def get_release_version_of_ref() -> semantic_version.Version: + cp = run(["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True) + if cp.returncode != 0: + raise RuntimeError("Failed to get current branch name") + branch_name = cp.stdout.strip() + ns, version_str = branch_name.split("/") + if ns != "rc": + raise RuntimeError("Not on a release branch!") + + try: + return semantic_version.Version(version_str) # type: ignore + except ValueError as e: + raise RuntimeError(f"Invalid version string: {e}") + +def main(args :'Namespace') -> Literal[1, 0]: + try: + release_version = semantic_version.Version(args.version) # type: ignore + if args.hotfix: + branch_release_version = get_release_version_of_ref() + expected_version = branch_release_version.next_patch() + if release_version != expected_version: + print(f"Error: Hotfix version `{release_version}` does not iterate on {branch_release_version}. Expected `{expected_version}`. ", file=stderr) + return 1 + return 0 + except ValueError as e: + print(f"Error: invalid version: {e}", file=stderr) + return 1 + except RuntimeError as e: + print(f"Error: {e}", file=stderr) + return 1 + +if __name__ == '__main__': + from sys import stderr, exit + import argparse + + parser = argparse.ArgumentParser(description="Validate a version string") + parser.add_argument("version", help="The version string to validate") + parser.add_argument('--hotfix', action='store_true', help="Whether the release is to hotfix an existing release.") + + exit(main(parser.parse_args())) \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 99348fe960..0ad0f1c747 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -9,8 +9,8 @@ MarkupSafe==1.1.1 requests==2.31.0 smmap==3.0.5 soupsieve==2.0.1 +pyyaml==6.0.1 urllib3==1.26.18 -pyyaml==5.4 wheel==0.38.1 jsonschema==4.9.1 marko==1.2.1 \ No newline at end of file