diff --git a/.github/ISSUE_TEMPLATE/join_org.md b/.github/ISSUE_TEMPLATE/join_org.md
index d12f9971..2e0b4ff1 100644
--- a/.github/ISSUE_TEMPLATE/join_org.md
+++ b/.github/ISSUE_TEMPLATE/join_org.md
@@ -13,16 +13,6 @@ Please read the [guide lines](https://www.kubeflow.org/docs/about/contributing/#
 
 ** Please list 2 existing members who are sponsoring your membership:**
 
-** Please test your PR **
-
-Run
-
-```
-cd github_orgs
-pytest test_org_yaml.py
-```
-Include the output in the PR
-
 **Additional Instructions**
 
 After your PR is merged please wait at least 1 hour for changes to propogate. 
diff --git a/.github/requirements.txt b/.github/requirements.txt
new file mode 100644
index 00000000..0db080b0
--- /dev/null
+++ b/.github/requirements.txt
@@ -0,0 +1,2 @@
+pytest==8.2.0
+PyYAML==6.0.1
\ No newline at end of file
diff --git a/.github/workflows/github_org_test.yaml b/.github/workflows/github_org_test.yaml
new file mode 100644
index 00000000..6f340e19
--- /dev/null
+++ b/.github/workflows/github_org_test.yaml
@@ -0,0 +1,29 @@
+name: Github Org Tests
+on:
+  pull_request:
+    paths:
+      - github-orgs/kubeflow/org.yaml
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: "3.10"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+        cache: "pip"
+
+    - name: Install PyTest
+      run: |
+        pip install -r .github/requirements.txt
+        pytest --version
+
+    - name: Run tests
+      working-directory: ./github-orgs
+      run: |
+        pytest test_org_yaml.py
diff --git a/README.md b/README.md
index 84a11acd..52fbaad2 100644
--- a/README.md
+++ b/README.md
@@ -4,8 +4,8 @@ Repository used to maintain group ACLs used by the Kubeflow community.
 
 For Google Groups in kubeflow.org, refer to `google_groups` subfolder.
 
-Run `pytest` after modifying `org.yaml`. Currently, this must be run
-manually. This test run automatically in a future change.
+To check if everything is fine after modifying `org.yaml`, you can run
+`pytest` locally from `github-orgs` directory.
 
 ## Joining Kubeflow GitHub organization
 
@@ -21,7 +21,6 @@ To join the Kubeflow GitHub organization, complete the following steps:
   cd github-orgs
   pytest test_org_yaml.py
   ```
-  Include the output in the PR.
 
 **Additional Instructions**