[HUDI-7438] Reimplement Azure CI report check with open PRs

.github/workflows/azure_ci.js

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+async function checkAzureCiAndCreateCommitStatus({ github, context, prNumber, latestCommitHash }) {
+  console.log(`- Checking Azure CI status of PR: ${prNumber} ${latestCommitHash}`);
+  const botUsername = 'hudi-bot';
+
+  const comments = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: prNumber,
+  });
+
+  // Find the latest comment from hudi-bot containing the Azure CI report
+  const botComments = comments.data.filter(comment => comment.user.login === botUsername);
+  const lastComment = botComments.pop();
+
+  let status = 'pending';
+  let message = 'In progress';
+  let azureRunLink = '';
+
+  if (lastComment) {
+    const reportPrefix = `${latestCommitHash} Azure: `
+    const successReportString = `${reportPrefix}[SUCCESS]`
+    const failureReportString = `${reportPrefix}[FAILURE]`
+
+    if (lastComment.body.includes(reportPrefix)) {
+      if (lastComment.body.includes(successReportString)) {
+        message = 'Successful on the latest commit';
+        status = 'success';
+      } else if (lastComment.body.includes(failureReportString)) {
+        message = 'Failed on the latest commit';
+        status = 'failure';
+      }
+    }
+
+    const linkRegex = /\[[a-zA-Z]+\]\((https?:\/\/[^\s]+)\)/;
+    const parts = lastComment.body.split(reportPrefix);
+    const secondPart = parts.length > 1 ? parts[1] : '';
+    const match = secondPart.match(linkRegex);
+
+    if (match) {
+      azureRunLink = match[1];
+    }
+  }
+
+  console.log(`Status: ${status}`);
+  console.log(`Azure Run Link: ${azureRunLink}`);
+  console.log(`${message}`);
+
+  console.log(`- Create commit status of PR based on Azure CI status: ${prNumber} ${latestCommitHash}`);
+  // Create or update the commit status for Azure CI
+  await github.rest.repos.createCommitStatus({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    sha: latestCommitHash,
+    state: status,
+    target_url: azureRunLink,
+    description: message,
+    context: 'Azure CI'
+  });
+
+  return { status, message, azureRunLink };
+}
+
+module.exports = checkAzureCiAndCreateCommitStatus;

.github/workflows/azure_ci_check.yml

@@ -27,67 +27,55 @@ permissions:
   issues: read
 
 jobs:
-  check-azure-ci-report:
-    if: "!contains(github.event.pull_request.body, 'HOTFIX: SKIP AZURE CI')"
+  check-azure-ci-and-add-commit-status:
+    if: |
+      github.event.issue.pull_request != null &&
+      github.event.issue.pull_request != '' &&
+      github.event.issue_comment.user.login == 'hudi-bot'
     runs-on: ubuntu-latest
     steps:
-      - name: Get last commit hash
-        id: last_commit
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const pr = context.payload.pull_request;
-            const lastCommitHash = pr.head.sha;
-            console.log(`Last commit hash: ${lastCommitHash}`);
-            // Set the output variable to be used in subsequent step
-            core.setOutput("last_commit_hash", lastCommitHash);
+      - name: Checkout repository
+        uses: actions/checkout@v2
 
-      - name: Check Azure CI report in PR comment
+      - name: Check PR state
+        id: check_pr_state
         uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const lastCommitHash = '${{ steps.last_commit.outputs.last_commit_hash }}'
-            const botUsername = 'hudi-bot';
-            
-            const issueNumber = context.payload.pull_request.number;
-            const comments = await github.rest.issues.listComments({
+            const issueNumber = github.event.issue.number;
+            const { data: pullRequest } = await github.rest.pulls.get({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: issueNumber,
+              pull_number: issueNumber
             });
             
-            // Find the last comment from hudi-bot containing the Azure CI report
-            const botComments = comments.data.filter(comment => comment.user.login === botUsername);
-            const lastComment = botComments.pop();
+            // Only check open PRs and a PR that is not a HOTFIX
+            const shouldSkip = (pullRequest.body.includes('HOTFIX: SKIP AZURE CI')
+              || pullRequest.state != 'open');
             
-            if (lastComment) {
-              const reportPrefix = '${lastCommitHash} Azure: '
-              const successReportString = '${reportPrefix}[SUCCESS]'
-              const failureReportString = '${reportPrefix}[FAILURE]'
-              if (lastComment.body.includes(reportPrefix)) {
-                if (lastComment.body.includes(successReportString)) {
-                  console.log(`Azure CI succeeded on the latest commit of the PR.`);
-                  return true;
-                } else if (lastComment.body.includes(failureReportString)) {
-                  console.log(`Azure CI failed on the latest commit of the PR.`);
-                  core.setFailed("Azure CI failed on the latest commit of the PR.");
-                  return false;
-                } else {
-                  console.log(`Azure CI is in progress on the latest commit of the PR.`);
-                  core.setFailed("Azure CI is in progress on the latest commit of the PR.");
-                  return false;
-                }
-              } else {
-                console.log(`No Azure CI report on the latest commit of the PR.`);
-                core.setFailed("No Azure CI report on the latest commit of the PR.");
-                return false;
-              }
-            } else {
-              console.log(`Azure CI report does not seem to be ready yet.`);
-              core.setFailed("Azure CI report does not seem to be ready yet.");
-              return false;
+            if (!shouldSkip) {
+              const commitHash = pullRequest.head.sha;
+              console.log(`Latest commit hash: ${commitHash}`);
+              // Set the output variable to be used in subsequent step
+              core.setOutput("latest_commit_hash", commitHash);
             }
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+            console.log(`Should skip Azure CI? ${shouldSkip}`);
+            return shouldSkip;
+
+      - name: Check Azure CI report and create commit status to PR
+        if: steps.check_pr_state.outputs.result != 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const latestCommitHash = '${{ steps.check_pr_state.outputs.latest_commit_hash }}'            
+            const issueNumber = github.event.issue.number;
+            const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`);
+            
+            await checkAzureCiAndCreateCommitStatus({
+              github,
+              context,
+              prNumber: issueNumber,
+              latestCommitHash: latestCommitHash
+            });

.github/workflows/scheduled_workflow.yml

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Scheduled Workflow
+
+on:
+  schedule:
+    # Runs every 5 minutes
+    - cron: '*/5 * * * *'
+
+permissions:
+  statuses: write
+  pull-requests: read
+  issues: read
+
+jobs:
+  process-new-and-updated-prs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Process new and updated PRs
+        # We have to run any actions that require write permissions here
+        # since the workflow triggered by events from a PR in a fork
+        # (not apache/hudi but other_owner/hudi) does not run on a
+        # GITHUB_TOKEN with write permissions (this is prohibited by
+        # Apache).
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const since = new Date(new Date().getTime() - (330 * 1000)).toISOString();
+            const query = `repo:${context.repo.owner}/${context.repo.repo} type:pr updated:>=${since}`;
+            const response = await github.rest.search.issuesAndPullRequests({
+              q: query
+            });
+            
+            // Filter for open PRs
+            const openPrs = response.data.items.filter(pr => pr.state === 'open');
+            const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`);
+            
+            for (const pr of openPrs) {
+              console.log(`*** Processing PR: ${pr.title}, URL: ${pr.html_url}`);
+            
+              if (!pr.body.includes('HOTFIX: SKIP AZURE CI')) {
+                const { data: pullRequest } = await github.rest.pulls.get({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  pull_number: pr.number
+                });
+                const latestCommitHash = pullRequest.head.sha;
+            
+                // Create commit status based on Azure CI report to PR
+                await checkAzureCiAndCreateCommitStatus({
+                  github,
+                  context,
+                  prNumber: pr.number,
+                  latestCommitHash: latestCommitHash
+                });
+              }
+            }