Rework upstream_utils scripts

.github/workflows/upstream-utils.yml

@@ -30,50 +30,61 @@ jobs:
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "Your Name"
-      - name: Run update_eigen.py
+      - name: Run eigen.py
         run: |
           cd upstream_utils
-          ./update_eigen.py
-      - name: Run update_fmt.py
+          ./eigen.py clone
+          ./eigen.py copy-upstream-to-thirdparty
+      - name: Run fmt.py
         run: |
           cd upstream_utils
-          ./update_fmt.py
-      - name: Run update_gcem.py
+          ./fmt.py clone
+          ./fmt.py copy-upstream-to-thirdparty
+      - name: Run gcem.py
         run: |
           cd upstream_utils
-          ./update_gcem.py
-      - name: Run update_json.py
+          ./gcem.py clone
+          ./gcem.py copy-upstream-to-thirdparty
+      - name: Run json.py
         run: |
           cd upstream_utils
-          ./update_json.py
-      - name: Run update_libuv.py
+          ./json.py clone
+          ./json.py copy-upstream-to-thirdparty
+      - name: Run libuv.py
         run: |
           cd upstream_utils
-          ./update_libuv.py
-      - name: Run update_llvm.py
+          ./libuv.py clone
+          ./libuv.py copy-upstream-to-thirdparty
+      - name: Run llvm.py
         run: |
           cd upstream_utils
-          ./update_llvm.py
-      - name: Run update_mpack.py
+          ./llvm.py clone
+          ./llvm.py copy-upstream-to-thirdparty
+      - name: Run mpack.py
         run: |
           cd upstream_utils
-          ./update_mpack.py
-      - name: Run update_stack_walker.py
+          ./mpack.py clone
+          ./mpack.py copy-upstream-to-thirdparty
+      - name: Run stack_walker.py
         run: |
           cd upstream_utils
-          ./update_stack_walker.py
-      - name: Run update_memory.py
+          ./stack_walker.py clone
+          ./stack_walker.py copy-upstream-to-thirdparty
+      - name: Run memory.py
         run: |
           cd upstream_utils
-          ./update_memory.py
-      - name: Run update_protobuf.py
+          ./memory.py clone
+          ./memory.py copy-upstream-to-thirdparty
+      - name: Run protobuf.py
         run: |
           cd upstream_utils
-          ./update_protobuf.py
-      - name: Run update_sleipnir.py
+          ./protobuf.py clone
+          ./protobuf.py copy-upstream-to-thirdparty
+      - name: Run sleipnir.py
         run: |
           cd upstream_utils
-          ./update_sleipnir.py
+          ./sleipnir.py clone
+          ./sleipnir.py copy-upstream-to-thirdparty
       - name: Add untracked files to index so they count as changes
         run: git add -A
       - name: Check output

upstream_utils/README.md

@@ -20,72 +20,37 @@ versions. Each library has its own patch directory (e.g., `lib_patches`).
 The example below will update a hypothetical library called `lib` to the tag
 `2.0`.
 
-Start in the `upstream_utils` folder. Restore the original repo.
+Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists.
 ```bash
-./update_<lib>.py
+./<lib>.py clone
 ```
 
-Navigate to the repo.
+Rebase the clone of the upstream repo.
 ```bash
-cd /tmp/lib
+./<lib>.py rebase 2.0
 ```
 
-Fetch the desired version using one of the following methods.
+Update the `upstream_utils` patch files and the tag in the script.
 ```bash
-# Fetch a full branch or tag
-git fetch origin 2.0
-
-# Fetch just a tag (useful for expensive-to-clone repos)
-git fetch --depth 1 origin tag 2.0
-```
-
-Rebase any patches onto the new version. If the old version and new version are
-on the same branch, run the following.
-```bash
-git rebase 2.0
+./<lib>.py format-patch 2.0
 ```
 
-If the old version and new version are on different branches (e.g.,
-llvm-project), use interactive rebase instead and remove commits that are common
-between the two branches from the list of commits to rebase. In other words,
-only commits representing downstream patches should be listed.
+Copy the updated upstream files into the thirdparty files within allwpilib.
 ```bash
-git rebase -i 2.0
-```
-
-Generate patch files for the new version.
-```bash
-git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature
-```
-
-Move the patch files to `upstream_utils`.
-```
-mv *.patch allwpilib/upstream_utils/lib_patches
-```
-
-Navigate back to `upstream_utils`.
-```bash
-cd allwpilib/upstream_utils
-```
-
-Modify the version number in the call to `setup_upstream_repo()` in
-`update_<lib>.py`, then rerun `update_<lib>.py` to reimport the thirdparty
-files.
-```bash
-./update_<lib>.py
+./<lib>.py copy-upstream-to-thirdparty
 ```
 
 ## Adding patch to thirdparty library
 
 The example below will add a new patch file to a hypothetical library called
 `lib` (Replace `<lib>` with `llvm`, `fmt`, `eigen`, ... in the following steps).
 
-Start in the `upstream_utils` folder. Restore the original repo.
+Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists.
 ```bash
-./update_<lib>.py
+./<lib>.py clone
 ```
 
-Navigate to the repo.
+Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command.
 ```bash
 cd /tmp/<lib>
 ```
@@ -96,24 +61,18 @@ git add ...
 git commit -m "..."
 ```
 
-Generate patch files.
+Update the `upstream_utils` patch files.
 ```bash
-git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature
-```
-where `2.0` is replaced with the version specified in `update_<lib>.py`.
-
-Move the patch files to `upstream_utils`.
-```
-mv *.patch allwpilib/upstream_utils/<lib>_patches
+allwpilib/upstream_utils/<lib>.py format-patch 2.0
 ```
+where `2.0` is replaced with the version specified in `<lib>.py`.
 
 Navigate back to `upstream_utils`.
 ```bash
 cd allwpilib/upstream_utils
 ```
 
-Update the list of patch files in `update_<lib>.py`, then rerun
-`update_<lib>.py` to reimport the thirdparty files.
+Update the list of patch files in `<lib>.py`, then rerun `<lib>.py` to reimport the thirdparty files.
 ```bash
-./update_<lib>.py
+./<lib>.py copy-upstream-to-thirdparty
 ```

upstream_utils/update_eigen.py → upstream_utils/eigen.py

@@ -5,11 +5,9 @@
 import shutil
 
 from upstream_utils import (
-    get_repo_root,
-    clone_repo,
     comment_out_invalid_includes,
     walk_cwd_and_copy_if,
-    git_am,
+    Lib,
 )
 
 
@@ -94,24 +92,9 @@ def unsupported_inclusions(dp, f):
     return "MatrixFunctions" in abspath
 
 
-def main():
-    upstream_root = clone_repo(
-        "https://gitlab.com/libeigen/eigen.git",
-        # master on 2024-05-22
-        "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198",
-        shallow=False,
-    )
-    wpilib_root = get_repo_root()
+def copy_upstream_src(wpilib_root):
     wpimath = os.path.join(wpilib_root, "wpimath")
 
-    # Apply patches to upstream Git repo
-    os.chdir(upstream_root)
-    for f in [
-        "0001-Disable-warnings.patch",
-        "0002-Intellisense-fix.patch",
-    ]:
-        git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f))
-
     # Delete old install
     for d in ["src/main/native/thirdparty/eigen/include"]:
         shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
@@ -138,10 +121,23 @@ def main():
         )
 
     shutil.copyfile(
-        os.path.join(upstream_root, ".clang-format"),
+        ".clang-format",
         os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"),
     )
 
 
+def main():
+    name = "eigen"
+    url = "https://gitlab.com/libeigen/eigen.git"
+    tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198"
+    patch_list = [
+        "0001-Disable-warnings.patch",
+        "0002-Intellisense-fix.patch",
+    ]
+
+    eigen = Lib(name, url, tag, patch_list, copy_upstream_src)
+    eigen.main()
+
+
 if __name__ == "__main__":
     main()

upstream_utils/update_expected.py → upstream_utils/expected.py

@@ -10,26 +10,18 @@
     comment_out_invalid_includes,
     walk_cwd_and_copy_if,
     git_am,
+    Lib,
 )
 
 
-def main():
-    upstream_root = clone_repo(
-        "https://github.com/TartanLlama/expected",
-        # master on 2024-01-25
-        "3f0ca7b19253129700a073abfa6d8638d9f7c80c",
-        shallow=False,
-    )
-    wpilib_root = get_repo_root()
+def copy_upstream_src(wpilib_root):
     wpiutil = os.path.join(wpilib_root, "wpiutil")
 
     # Copy expected header into allwpilib
     dest_filename = os.path.join(
         wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected"
     )
-    shutil.copyfile(
-        os.path.join(upstream_root, "include/tl/expected.hpp"), dest_filename
-    )
+    shutil.copyfile("include/tl/expected.hpp", dest_filename)
 
     # Rename namespace from tl to wpi
     with open(dest_filename) as f:
@@ -41,5 +33,15 @@ def main():
         f.write(content)
 
 
+def main():
+    name = "expected"
+    url = "https://github.com/TartanLlama/expected"
+    # master on 2024-01-25
+    tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c"
+
+    expected = Lib(name, url, tag, [], copy_upstream_src)
+    expected.main()
+
+
 if __name__ == "__main__":
     main()

upstream_utils/update_fmt.py → upstream_utils/fmt.py

@@ -9,19 +9,13 @@
     comment_out_invalid_includes,
     walk_cwd_and_copy_if,
     git_am,
+    Lib,
 )
 
 
-def main():
-    upstream_root = clone_repo("https://github.com/fmtlib/fmt", "11.0.1")
-    wpilib_root = get_repo_root()
+def copy_upstream_src(wpilib_root):
     wpiutil = os.path.join(wpilib_root, "wpiutil")
 
-    # Apply patches to upstream Git repo
-    os.chdir(upstream_root)
-    for f in ["0001-Suppress-warnings-we-can-t-fix.patch"]:
-        git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f))
-
     # Delete old install
     for d in [
         "src/main/native/thirdparty/fmtlib/src",
@@ -51,5 +45,16 @@ def main():
         )
 
 
+def main():
+    name = "fmt"
+    url = "https://github.com/fmtlib/fmt"
+    tag = "11.0.1"
+
+    patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"]
+
+    fmt = Lib(name, url, tag, patch_list, copy_upstream_src)
+    fmt.main()
+
+
 if __name__ == "__main__":
     main()

upstream_utils/update_gcem.py → upstream_utils/gcem.py

@@ -9,22 +9,13 @@
     comment_out_invalid_includes,
     walk_cwd_and_copy_if,
     git_am,
+    Lib,
 )
 
 
-def main():
-    upstream_root = clone_repo("https://github.com/kthohr/gcem.git", "v1.18.0")
-    wpilib_root = get_repo_root()
+def copy_upstream_src(wpilib_root):
     wpimath = os.path.join(wpilib_root, "wpimath")
 
-    # Apply patches to upstream Git repo
-    os.chdir(upstream_root)
-    for f in [
-        "0001-Call-std-functions-if-not-constant-evaluated.patch",
-        "0002-Add-hypot-x-y-z.patch",
-    ]:
-        git_am(os.path.join(wpilib_root, "upstream_utils/gcem_patches", f))
-
     # Delete old install
     for d in [
         "src/main/native/thirdparty/gcem/include",
@@ -43,5 +34,19 @@ def main():
         )
 
 
+def main():
+    name = "gcem"
+    url = "https://github.com/kthohr/gcem.git"
+    tag = "v1.18.0"
+
+    patch_list = [
+        "0001-Call-std-functions-if-not-constant-evaluated.patch",
+        "0002-Add-hypot-x-y-z.patch",
+    ]
+
+    gcem = Lib(name, url, tag, patch_list, copy_upstream_src)
+    gcem.main()
+
+
 if __name__ == "__main__":
     main()