diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml
index 2f7b54d2a67e90..db293e2d2f5883 100644
--- a/.github/actions/setup-sentry/action.yml
+++ b/.github/actions/setup-sentry/action.yml
@@ -111,9 +111,6 @@ runs:
         pip install -U -e ".[dev]"
         cd -
 
-        # pytest plugin used for better pytest failure annotations
-        pip install pytest-github-actions-annotate-failures
-
     - name: Start devservices
       shell: bash
       env:
diff --git a/conftest.py b/conftest.py
index d0289ae9c1e99c..f958672c328e45 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,6 @@
 import os
 import sys
+from collections import OrderedDict
 
 import pytest
 
@@ -27,3 +28,95 @@ def pytest_addoption(parser):
 def pytest_runtest_setup(item):
     if item.get_closest_marker("itunes") and not item.config.getoption("--itunes"):
         pytest.skip("Test requires --itunes")
+
+
+# XXX: The below code is vendored code from https://github.com/utgwkk/pytest-github-actions-annotate-failures
+# so that we can add support for pytest_rerunfailures
+# retried tests will no longer be annotated in GHA
+#
+# Reference:
+# https://docs.pytest.org/en/latest/writing_plugins.html#hookwrapper-executing-around-other-hooks
+# https://docs.pytest.org/en/latest/writing_plugins.html#hook-function-ordering-call-example
+# https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_runtest_makereport
+#
+# Inspired by:
+# https://github.com/pytest-dev/pytest/blob/master/src/_pytest/terminal.py
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    report = outcome.get_result()
+
+    # enable only in a workflow of GitHub Actions
+    # ref: https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables
+    if os.environ.get("GITHUB_ACTIONS") != "true":
+        return
+
+    try:
+        # If we have the pytest_rerunfailures plugin,
+        # and there are still retries to be run,
+        # then do not return the error
+        import pytest_rerunfailures
+
+        if item.execution_count <= pytest_rerunfailures.get_reruns_count(item):
+            return
+    except ImportError:
+        pass
+
+    if report.when == "call" and report.failed:
+        # collect information to be annotated
+        filesystempath, lineno, _ = report.location
+
+        # try to convert to absolute path in GitHub Actions
+        workspace = os.environ.get("GITHUB_WORKSPACE")
+        if workspace:
+            full_path = os.path.abspath(filesystempath)
+            try:
+                rel_path = os.path.relpath(full_path, workspace)
+            except ValueError:
+                # os.path.relpath() will raise ValueError on Windows
+                # when full_path and workspace have different mount points.
+                # https://github.com/utgwkk/pytest-github-actions-annotate-failures/issues/20
+                rel_path = filesystempath
+            if not rel_path.startswith(".."):
+                filesystempath = rel_path
+
+        # 0-index to 1-index
+        lineno += 1
+
+        # get the name of the current failed test, with parametrize info
+        longrepr = report.head_line or item.name
+
+        # get the error message and line number from the actual error
+        try:
+            longrepr += "\n\n" + report.longrepr.reprcrash.message
+            lineno = report.longrepr.reprcrash.lineno
+
+        except AttributeError:
+            pass
+
+        print(  # noqa: B314
+            _error_workflow_command(filesystempath, lineno, longrepr), file=sys.stderr
+        )
+
+
+def _error_workflow_command(filesystempath, lineno, longrepr):
+    # Build collection of arguments. Ordering is strict for easy testing
+    details_dict = OrderedDict()
+    details_dict["file"] = filesystempath
+    if lineno is not None:
+        details_dict["line"] = lineno
+
+    details = ",".join(f"{k}={v}" for k, v in details_dict.items())
+
+    if longrepr is None:
+        return f"\n::error {details}"
+    else:
+        longrepr = _escape(longrepr)
+        return f"\n::error {details}::{longrepr}"
+
+
+def _escape(s):
+    return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")