From 00d91d2d6f494f1b8aa10cdaa2cbad0342f2b041 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sat, 13 Jul 2024 15:48:25 -0700 Subject: [PATCH] Rework upstream_utils scripts --- .github/workflows/upstream-utils.yml | 55 +++-- upstream_utils/README.md | 73 ++----- upstream_utils/{update_eigen.py => eigen.py} | 36 ++-- .../{update_expected.py => expected.py} | 24 ++- upstream_utils/{update_fmt.py => fmt.py} | 21 +- upstream_utils/{update_gcem.py => gcem.py} | 27 ++- upstream_utils/{update_json.py => json.py} | 43 ++-- upstream_utils/{update_libuv.py => libuv.py} | 43 ++-- upstream_utils/{update_llvm.py => llvm.py} | 32 +-- .../{update_memory.py => memory.py} | 17 +- upstream_utils/{update_mpack.py => mpack.py} | 34 +-- .../{update_protobuf.py => protobuf.py} | 50 ++--- .../{update_sleipnir.py => sleipnir.py} | 42 ++-- ...update_stack_walker.py => stack_walker.py} | 54 ++--- upstream_utils/upstream_utils.py | 193 ++++++++++++++++++ 15 files changed, 473 insertions(+), 271 deletions(-) rename upstream_utils/{update_eigen.py => eigen.py} (86%) rename upstream_utils/{update_expected.py => expected.py} (69%) rename upstream_utils/{update_fmt.py => fmt.py} (80%) rename upstream_utils/{update_gcem.py => gcem.py} (76%) rename upstream_utils/{update_json.py => json.py} (78%) rename upstream_utils/{update_libuv.py => libuv.py} (86%) rename upstream_utils/{update_llvm.py => llvm.py} (96%) rename upstream_utils/{update_memory.py => memory.py} (91%) rename upstream_utils/{update_mpack.py => mpack.py} (80%) rename upstream_utils/{update_protobuf.py => protobuf.py} (97%) rename upstream_utils/{update_sleipnir.py => sleipnir.py} (76%) rename upstream_utils/{update_stack_walker.py => stack_walker.py} (64%) diff --git a/.github/workflows/upstream-utils.yml b/.github/workflows/upstream-utils.yml index 102aaf621bc..e9c04a45823 100644 --- a/.github/workflows/upstream-utils.yml +++ b/.github/workflows/upstream-utils.yml @@ -30,50 +30,61 @@ jobs: run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - - name: Run update_eigen.py + - name: Run eigen.py run: | cd upstream_utils - ./update_eigen.py - - name: Run update_fmt.py + ./eigen.py clone + ./eigen.py copy-upstream-to-thirdparty + - name: Run fmt.py run: | cd upstream_utils - ./update_fmt.py - - name: Run update_gcem.py + ./fmt.py clone + ./fmt.py copy-upstream-to-thirdparty + - name: Run gcem.py run: | cd upstream_utils - ./update_gcem.py - - name: Run update_json.py + ./gcem.py clone + ./gcem.py copy-upstream-to-thirdparty + - name: Run json.py run: | cd upstream_utils - ./update_json.py - - name: Run update_libuv.py + ./json.py clone + ./json.py copy-upstream-to-thirdparty + - name: Run libuv.py run: | cd upstream_utils - ./update_libuv.py - - name: Run update_llvm.py + ./libuv.py clone + ./libuv.py copy-upstream-to-thirdparty + - name: Run llvm.py run: | cd upstream_utils - ./update_llvm.py - - name: Run update_mpack.py + ./llvm.py clone + ./llvm.py copy-upstream-to-thirdparty + - name: Run mpack.py run: | cd upstream_utils - ./update_mpack.py - - name: Run update_stack_walker.py + ./mpack.py clone + ./mpack.py copy-upstream-to-thirdparty + - name: Run stack_walker.py run: | cd upstream_utils - ./update_stack_walker.py - - name: Run update_memory.py + ./stack_walker.py clone + ./stack_walker.py copy-upstream-to-thirdparty + - name: Run memory.py run: | cd upstream_utils - ./update_memory.py - - name: Run update_protobuf.py + ./memory.py clone + ./memory.py copy-upstream-to-thirdparty + - name: Run protobuf.py run: | cd upstream_utils - ./update_protobuf.py - - name: Run update_sleipnir.py + ./protobuf.py clone + ./protobuf.py copy-upstream-to-thirdparty + - name: Run sleipnir.py run: | cd upstream_utils - ./update_sleipnir.py + ./sleipnir.py clone + ./sleipnir.py copy-upstream-to-thirdparty - name: Add untracked files to index so they count as changes run: git add -A - name: Check output diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 979dd0d6727..81a33af8cf6 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -20,59 +20,24 @@ versions. Each library has its own patch directory (e.g., `lib_patches`). The example below will update a hypothetical library called `lib` to the tag `2.0`. -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Rebase the clone of the upstream repo. ```bash -cd /tmp/lib +./.py rebase 2.0 ``` -Fetch the desired version using one of the following methods. +Update the `upstream_utils` patch files and the tag in the script. ```bash -# Fetch a full branch or tag -git fetch origin 2.0 - -# Fetch just a tag (useful for expensive-to-clone repos) -git fetch --depth 1 origin tag 2.0 -``` - -Rebase any patches onto the new version. If the old version and new version are -on the same branch, run the following. -```bash -git rebase 2.0 +./.py format-patch 2.0 ``` -If the old version and new version are on different branches (e.g., -llvm-project), use interactive rebase instead and remove commits that are common -between the two branches from the list of commits to rebase. In other words, -only commits representing downstream patches should be listed. +Copy the updated upstream files into the thirdparty files within allwpilib. ```bash -git rebase -i 2.0 -``` - -Generate patch files for the new version. -```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/lib_patches -``` - -Navigate back to `upstream_utils`. -```bash -cd allwpilib/upstream_utils -``` - -Modify the version number in the call to `setup_upstream_repo()` in -`update_.py`, then rerun `update_.py` to reimport the thirdparty -files. -```bash -./update_.py +./.py copy-upstream-to-thirdparty ``` ## Adding patch to thirdparty library @@ -80,12 +45,12 @@ files. The example below will add a new patch file to a hypothetical library called `lib` (Replace `` with `llvm`, `fmt`, `eigen`, ... in the following steps). -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command. ```bash cd /tmp/ ``` @@ -96,24 +61,18 @@ git add ... git commit -m "..." ``` -Generate patch files. +Update the `upstream_utils` patch files. ```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` -where `2.0` is replaced with the version specified in `update_.py`. - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/_patches +allwpilib/upstream_utils/.py format-patch 2.0 ``` +where `2.0` is replaced with the version specified in `.py`. Navigate back to `upstream_utils`. ```bash cd allwpilib/upstream_utils ``` -Update the list of patch files in `update_.py`, then rerun -`update_.py` to reimport the thirdparty files. +Update the list of patch files in `.py`, then rerun `.py` to reimport the thirdparty files. ```bash -./update_.py +./.py copy-upstream-to-thirdparty ``` diff --git a/upstream_utils/update_eigen.py b/upstream_utils/eigen.py similarity index 86% rename from upstream_utils/update_eigen.py rename to upstream_utils/eigen.py index fd8cfb4ef60..cc85f65e414 100755 --- a/upstream_utils/update_eigen.py +++ b/upstream_utils/eigen.py @@ -5,11 +5,9 @@ import shutil from upstream_utils import ( - get_repo_root, - clone_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, - git_am, + Lib, ) @@ -94,24 +92,9 @@ def unsupported_inclusions(dp, f): return "MatrixFunctions" in abspath -def main(): - upstream_root = clone_repo( - "https://gitlab.com/libeigen/eigen.git", - # master on 2024-05-22 - "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Disable-warnings.patch", - "0002-Intellisense-fix.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/eigen/include"]: shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) @@ -138,10 +121,23 @@ def main(): ) shutil.copyfile( - os.path.join(upstream_root, ".clang-format"), + ".clang-format", os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"), ) +def main(): + name = "eigen" + url = "https://gitlab.com/libeigen/eigen.git" + tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198" + patch_list = [ + "0001-Disable-warnings.patch", + "0002-Intellisense-fix.patch", + ] + + eigen = Lib(name, url, tag, patch_list, copy_upstream_src) + eigen.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_expected.py b/upstream_utils/expected.py similarity index 69% rename from upstream_utils/update_expected.py rename to upstream_utils/expected.py index 1b459fe5ac3..08569e3561b 100755 --- a/upstream_utils/update_expected.py +++ b/upstream_utils/expected.py @@ -10,26 +10,18 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/TartanLlama/expected", - # master on 2024-01-25 - "3f0ca7b19253129700a073abfa6d8638d9f7c80c", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Copy expected header into allwpilib dest_filename = os.path.join( wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected" ) - shutil.copyfile( - os.path.join(upstream_root, "include/tl/expected.hpp"), dest_filename - ) + shutil.copyfile("include/tl/expected.hpp", dest_filename) # Rename namespace from tl to wpi with open(dest_filename) as f: @@ -41,5 +33,15 @@ def main(): f.write(content) +def main(): + name = "expected" + url = "https://github.com/TartanLlama/expected" + # master on 2024-01-25 + tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c" + + expected = Lib(name, url, tag, [], copy_upstream_src) + expected.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_fmt.py b/upstream_utils/fmt.py similarity index 80% rename from upstream_utils/update_fmt.py rename to upstream_utils/fmt.py index 94aef193f71..8f03859f54c 100755 --- a/upstream_utils/update_fmt.py +++ b/upstream_utils/fmt.py @@ -9,19 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/fmtlib/fmt", "11.0.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in ["0001-Suppress-warnings-we-can-t-fix.patch"]: - git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/fmtlib/src", @@ -51,5 +45,16 @@ def main(): ) +def main(): + name = "fmt" + url = "https://github.com/fmtlib/fmt" + tag = "11.0.1" + + patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"] + + fmt = Lib(name, url, tag, patch_list, copy_upstream_src) + fmt.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_gcem.py b/upstream_utils/gcem.py similarity index 76% rename from upstream_utils/update_gcem.py rename to upstream_utils/gcem.py index 20f62425962..b6fd472a72f 100755 --- a/upstream_utils/update_gcem.py +++ b/upstream_utils/gcem.py @@ -9,22 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/kthohr/gcem.git", "v1.18.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Call-std-functions-if-not-constant-evaluated.patch", - "0002-Add-hypot-x-y-z.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/gcem_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/gcem/include", @@ -43,5 +34,19 @@ def main(): ) +def main(): + name = "gcem" + url = "https://github.com/kthohr/gcem.git" + tag = "v1.18.0" + + patch_list = [ + "0001-Call-std-functions-if-not-constant-evaluated.patch", + "0002-Add-hypot-x-y-z.patch", + ] + + gcem = Lib(name, url, tag, patch_list, copy_upstream_src) + gcem.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_json.py b/upstream_utils/json.py similarity index 78% rename from upstream_utils/update_json.py rename to upstream_utils/json.py index 9cad98d5fa3..78924ef2a1e 100755 --- a/upstream_utils/update_json.py +++ b/upstream_utils/json.py @@ -8,27 +8,13 @@ clone_repo, walk_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/nlohmann/json", "v3.11.3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-version-from-namespace.patch", - "0002-Make-serializer-public.patch", - "0003-Make-dump_escaped-take-std-string_view.patch", - "0004-Add-llvm-stream-support.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/json_patches", f), - use_threeway=True, - ) - # Delete old install for d in [ "src/main/native/thirdparty/json/include", @@ -36,11 +22,9 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Create lists of source and destination files - os.chdir(os.path.join(upstream_root, "include/nlohmann")) + os.chdir("include/nlohmann") files = walk_if(".", lambda dp, f: True) - src_include_files = [ - os.path.join(os.path.join(upstream_root, "include/nlohmann"), f) for f in files - ] + src_include_files = [os.path.abspath(f) for f in files] wpiutil_json_root = os.path.join( wpiutil, "src/main/native/thirdparty/json/include/wpi" ) @@ -74,5 +58,24 @@ def main(): f.write(content) +def main(): + name = "json" + url = "https://github.com/nlohmann/json" + tag = "v3.11.3" + + patch_list = [ + "0001-Remove-version-from-namespace.patch", + "0002-Make-serializer-public.patch", + "0003-Make-dump_escaped-take-std-string_view.patch", + "0004-Add-llvm-stream-support.patch", + ] + patch_options = { + "use_threeway": True, + } + + json = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + json.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_libuv.py b/upstream_utils/libuv.py similarity index 86% rename from upstream_utils/update_libuv.py rename to upstream_utils/libuv.py index 0290c4d73a8..9a2865838cf 100755 --- a/upstream_utils/update_libuv.py +++ b/upstream_utils/libuv.py @@ -9,30 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/libuv/libuv", "v1.48.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpinet = os.path.join(wpilib_root, "wpinet") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", - "0002-Fix-missing-casts.patch", - "0003-Fix-warnings.patch", - "0004-Preprocessor-cleanup.patch", - "0005-Cleanup-problematic-language.patch", - "0006-Fix-Win32-warning-suppression-pragma.patch", - "0007-Use-C-atomics.patch", - "0008-Remove-static-from-array-indices.patch", - "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", - "0010-Remove-swearing.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/libuv_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/libuv"]: shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True) @@ -71,5 +54,27 @@ def main(): ) +def main(): + name = "libuv" + url = "https://github.com/libuv/libuv" + tag = "v1.48.0" + + patch_list = [ + "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", + "0002-Fix-missing-casts.patch", + "0003-Fix-warnings.patch", + "0004-Preprocessor-cleanup.patch", + "0005-Cleanup-problematic-language.patch", + "0006-Fix-Win32-warning-suppression-pragma.patch", + "0007-Use-C-atomics.patch", + "0008-Remove-static-from-array-indices.patch", + "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", + "0010-Remove-swearing.patch", + ] + + libuv = Lib(name, url, tag, patch_list, copy_upstream_src) + libuv.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_llvm.py b/upstream_utils/llvm.py similarity index 96% rename from upstream_utils/update_llvm.py rename to upstream_utils/llvm.py index 9180442a631..394d3cb7a5d 100755 --- a/upstream_utils/update_llvm.py +++ b/upstream_utils/llvm.py @@ -9,6 +9,7 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) @@ -170,14 +171,20 @@ def overwrite_tests(wpiutil_root, llvm_root): run_global_replacements(wpi_files) -def main(): - upstream_root = clone_repo("https://github.com/llvm/llvm-project", "llvmorg-18.1.8") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ + overwrite_source(wpiutil, upstream_root) + overwrite_tests(wpiutil, upstream_root) + + +def main(): + name = "llvm" + url = "https://github.com/llvm/llvm-project" + tag = "llvmorg-18.1.8" + + patch_list = [ "0001-Remove-StringRef-ArrayRef-and-Optional.patch", "0002-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch", "0003-Change-unique_function-storage-size.patch", @@ -214,14 +221,13 @@ def main(): "0034-Add-back-removed-raw_string_ostream-write_impl.patch", "0035-Remove-auto-conversion-from-raw_ostream.patch", "0036-Add-SmallVector-erase_if.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/llvm_patches", f), - use_threeway=True, - ) + ] + patch_options = { + "use_threeway": True, + } - overwrite_source(wpiutil, upstream_root) - overwrite_tests(wpiutil, upstream_root) + llvm = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + llvm.main() if __name__ == "__main__": diff --git a/upstream_utils/update_memory.py b/upstream_utils/memory.py similarity index 91% rename from upstream_utils/update_memory.py rename to upstream_utils/memory.py index 6645aaf60cf..4a618584220 100755 --- a/upstream_utils/update_memory.py +++ b/upstream_utils/memory.py @@ -9,6 +9,7 @@ comment_out_invalid_includes, walk_if, copy_to, + Lib, ) @@ -58,9 +59,7 @@ def run_global_replacements(memory_files): f.write(content) -def main(): - upstream_root = clone_repo("https://github.com/foonathan/memory", "v0.7-3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -71,7 +70,6 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Copy sources - os.chdir(upstream_root) src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp")) src_files = copy_to( src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory") @@ -80,7 +78,7 @@ def main(): run_source_replacements(src_files) # Copy headers - os.chdir(os.path.join(upstream_root, "include", "foonathan")) + os.chdir(os.path.join("include", "foonathan")) include_files = walk_if(".", lambda dp, f: f.endswith(".hpp")) include_files = copy_to( include_files, @@ -100,5 +98,14 @@ def main(): ) +def main(): + name = "memory" + url = "https://github.com/foonathan/memory" + tag = "v0.7-3" + + memory = Lib(name, url, tag, [], copy_upstream_src) + memory.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_mpack.py b/upstream_utils/mpack.py similarity index 80% rename from upstream_utils/update_mpack.py rename to upstream_utils/mpack.py index 68315e933a9..678b189d476 100755 --- a/upstream_utils/update_mpack.py +++ b/upstream_utils/mpack.py @@ -9,12 +9,11 @@ clone_repo, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/ludocode/mpack", "v1.1.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -24,19 +23,6 @@ def main(): ]: shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) - # Apply patches to upstream Git repo - os.chdir(upstream_root) - - for f in [ - "0001-Don-t-emit-inline-defs.patch", - "0002-Update-amalgamation-script.patch", - "0003-Use-namespace-for-C.patch", - "0004-Group-doxygen-into-MPack-module.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/mpack_patches", f), - ) - # Run the amalgmation script subprocess.check_call(["bash", "tools/amalgamate.sh"]) @@ -56,5 +42,21 @@ def main(): ) +def main(): + name = "mpack" + url = "https://github.com/ludocode/mpack" + tag = "v1.1.1" + + patch_list = [ + "0001-Don-t-emit-inline-defs.patch", + "0002-Update-amalgamation-script.patch", + "0003-Use-namespace-for-C.patch", + "0004-Group-doxygen-into-MPack-module.patch", + ] + + mpack = Lib(name, url, tag, [], copy_upstream_src) + mpack.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_protobuf.py b/upstream_utils/protobuf.py similarity index 97% rename from upstream_utils/update_protobuf.py rename to upstream_utils/protobuf.py index 1f004cd0c42..8b2adf5347b 100755 --- a/upstream_utils/update_protobuf.py +++ b/upstream_utils/protobuf.py @@ -11,6 +11,7 @@ walk_cwd_and_copy_if, walk_if, git_am, + Lib, ) protobuf_lite_sources = set( @@ -255,31 +256,10 @@ def matches(dp, f, files): return p in files -def main(): - upstream_root = clone_repo( - "https://github.com/protocolbuffers/protobuf", "v3.21.12" - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Fix-sign-compare-warnings.patch", - "0002-Remove-redundant-move.patch", - "0003-Fix-maybe-uninitialized-warnings.patch", - "0004-Fix-coded_stream-WriteRaw.patch", - "0005-Suppress-enum-enum-conversion-warning.patch", - "0006-Fix-noreturn-function-returning.patch", - "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", - "0008-Disable-MSVC-switch-warning.patch", - "0009-Disable-unused-function-warning.patch", - "0010-Disable-pedantic-warning.patch", - "0011-Avoid-use-of-sprintf.patch", - "0012-Suppress-stringop-overflow-warning-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/protobuf_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/protobuf/src", @@ -304,5 +284,29 @@ def main(): ) +def main(): + name = "protobuf" + url = "https://github.com/protocolbuffers/protobuf" + tag = "v3.21.12" + + patch_list = [ + "0001-Fix-sign-compare-warnings.patch", + "0002-Remove-redundant-move.patch", + "0003-Fix-maybe-uninitialized-warnings.patch", + "0004-Fix-coded_stream-WriteRaw.patch", + "0005-Suppress-enum-enum-conversion-warning.patch", + "0006-Fix-noreturn-function-returning.patch", + "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", + "0008-Disable-MSVC-switch-warning.patch", + "0009-Disable-unused-function-warning.patch", + "0010-Disable-pedantic-warning.patch", + "0011-Avoid-use-of-sprintf.patch", + "0012-Suppress-stringop-overflow-warning-false-positives.patch", + ] + + protobuf = Lib(name, url, tag, patch_list, copy_upstream_src) + protobuf.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_sleipnir.py b/upstream_utils/sleipnir.py similarity index 76% rename from upstream_utils/update_sleipnir.py rename to upstream_utils/sleipnir.py index 4b783c82bab..11b20cc29a8 100755 --- a/upstream_utils/update_sleipnir.py +++ b/upstream_utils/sleipnir.py @@ -9,30 +9,13 @@ copy_to, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/SleipnirGroup/Sleipnir", - # main on 2024-07-09 - "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-using-enum-declarations.patch", - "0002-Use-fmtlib.patch", - "0003-Remove-unsupported-constexpr.patch", - "0004-Use-wpi-SmallVector.patch", - "0005-Suppress-clang-tidy-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/sleipnir_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/sleipnir/src", @@ -41,7 +24,6 @@ def main(): shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) # Copy Sleipnir source files into allwpilib - os.chdir(upstream_root) src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn] src_files = copy_to( src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir") @@ -65,10 +47,28 @@ def main(): ".styleguide-license", ]: shutil.copyfile( - os.path.join(upstream_root, filename), + filename, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename), ) +def main(): + name = "sleipnir" + url = "https://github.com/SleipnirGroup/Sleipnir" + # main on 2024-07-09 + tag = "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5" + + patch_list = [ + "0001-Remove-using-enum-declarations.patch", + "0002-Use-fmtlib.patch", + "0003-Remove-unsupported-constexpr.patch", + "0004-Use-wpi-SmallVector.patch", + "0005-Suppress-clang-tidy-false-positives.patch", + ] + + sleipnir = Lib(name, url, tag, patch_list, copy_upstream_src) + sleipnir.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_stack_walker.py b/upstream_utils/stack_walker.py similarity index 64% rename from upstream_utils/update_stack_walker.py rename to upstream_utils/stack_walker.py index a58886ec799..190f12d04d9 100755 --- a/upstream_utils/update_stack_walker.py +++ b/upstream_utils/stack_walker.py @@ -10,11 +10,12 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def crlf_to_lf(stackwalker_dir): - for root, _, files in os.walk(stackwalker_dir): +def crlf_to_lf(): + for root, _, files in os.walk("."): if ".git" in root: continue @@ -28,35 +29,13 @@ def crlf_to_lf(stackwalker_dir): with open(filename, "wb") as f: f.write(content) - cwd = os.getcwd() - os.chdir(stackwalker_dir) subprocess.check_call(["git", "add", "-A"]) subprocess.check_call(["git", "commit", "-m", "Fix line endings"]) - os.chdir(cwd) -def main(): - upstream_root = clone_repo( - "https://github.com/JochenKalmbach/StackWalker", - "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Run CRLF -> LF before trying any patches - crlf_to_lf(upstream_root) - - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Add-advapi-pragma.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/stack_walker_patches", f), - ignore_whitespace=True, - ) - shutil.copy( os.path.join("Main", "StackWalker", "StackWalker.h"), os.path.join(wpiutil, "src/main/native/windows/StackWalker.h"), @@ -68,5 +47,30 @@ def main(): ) +def main(): + name = "stack_walker" + url = "https://github.com/JochenKalmbach/StackWalker" + tag = "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b" + + patch_list = [ + "0001-Add-advapi-pragma.patch", + ] + patch_options = { + "ignore_whitespace": True, + } + + stack_walker = Lib( + name, + url, + tag, + patch_list, + copy_upstream_src, + patch_options, + pre_patch_hook=crlf_to_lf, + pre_patch_commits=1, + ) + stack_walker.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index ccf668b738f..19af3b86ff5 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -1,7 +1,9 @@ +import argparse import os import re import shutil import subprocess +import sys import tempfile @@ -198,3 +200,194 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): args.append("--ignore-whitespace") subprocess.check_output(args + [patch]) + + +def has_git_rev(rev): + cmd = ["git", "rev-parse", "--verify", "-q", rev] + return subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode == 0 + + +class Lib: + def __init__( + self, + name, + url, + old_tag, + patch_list, + copy_upstream_src, + patch_options={}, + *, + pre_patch_hook=None, + pre_patch_commits=0, + ): + self.name = name + self.url = url + self.old_tag = old_tag + self.patch_list = patch_list + self.copy_upstream_src = copy_upstream_src + self.patch_options = patch_options + self.pre_patch_hook = pre_patch_hook + self.pre_patch_commits = pre_patch_commits + self.wpilib_root = get_repo_root() + + def open_repo(self, *, err_msg_if_absent): + os.chdir(tempfile.gettempdir()) + + repo = os.path.basename(self.url) + dest = os.path.join(os.getcwd(), repo).removesuffix(".git") + + print(f"INFO: Opening repository at {dest}") + + if not os.path.exists(dest): + if err_msg_if_absent is None: + subprocess.run(["git", "clone", "--filter=tree:0", self.url]) + else: + print(err_msg_if_absent, file=sys.stderr) + exit(1) + os.chdir(dest) + + def replace_tag(self, tag): + path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") + with open(path, "r") as file: + lines = file.readlines() + + previous_text = f"tag = '{self.old_tag}'" + new_text = f"tag = '{tag}'" + for i in range(len(lines)): + lines[i] = lines[i].replace(previous_text, new_text) + + with open(path, "w") as file: + file.writelines(lines) + + def clone(self): + self.open_repo(err_msg_if_absent=None) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + def rebase(self, new_tag): + self.open_repo( + err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' + ) + + subprocess.run(["git", "fetch", "origin", new_tag]) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + + subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag]) + + # Detect merge conflict by detecting if we stopped in the middle of a rebase + if has_git_rev("REBASE_HEAD"): + print( + f"Merge conflicts when rebasing onto {new_tag}! You must manually resolve them.", + file=sys.stderr, + ) + + def format_patch(self, tag): + self.open_repo( + err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' + ) + + if self.pre_patch_commits > 0: + commits_since_tag_output = subprocess.run( + ["git", "log", "--format=format:%h", f"{tag}..HEAD"], + capture_output=True, + ).stdout + commits_since_tag = commits_since_tag_output.count(b"\n") + 1 + tag = f"HEAD~{commits_since_tag - self.pre_patch_commits}" + + subprocess.run( + [ + "git", + "format-patch", + f"{tag}..HEAD", + "--abbrev=40", + "--zero-commit", + "--no-signature", + ] + ) + + patch_dest = os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches" + ) + + if not os.path.exists(patch_dest): + print( + f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr + ) + + for f in os.listdir(): + if f.endswith(".patch"): + os.rename(f, os.path.join(patch_dest, f)) + + self.replace_tag(tag) + + def copy_upstream_to_thirdparty(self): + self.open_repo( + err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' + ) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + + self.copy_upstream_src(self.wpilib_root) + + def main(self, argv=sys.argv[1:]): + parser = argparse.ArgumentParser( + description=f"CLI manager of the {self.name} upstream library" + ) + subparsers = parser.add_subparsers(dest="subcommand", required=True) + + subparsers.add_parser( + "clone", help="Clones the upstream repository in a local tempdir" + ) + + parser_rebase = subparsers.add_parser( + "rebase", help="Rebases the clone of the upstream repository" + ) + parser_rebase.add_argument("new_tag", help="The tag to rebase onto") + + parser_format_patch = subparsers.add_parser( + "format-patch", + help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory", + ) + parser_format_patch.add_argument( + "new_tag", help="The tag for the commit before the patches" + ) + + subparsers.add_parser( + "copy-upstream-to-thirdparty", + help="Copies files from the upstream repository into the thirdparty directory in allwpilib", + ) + + args = parser.parse_args(argv) + + self.wpilib_root = get_repo_root() + if args.subcommand == "clone": + self.clone() + elif args.subcommand == "rebase": + self.rebase(args.new_tag) + elif args.subcommand == "format-patch": + self.format_patch(args.new_tag) + elif args.subcommand == "copy-upstream-to-thirdparty": + self.copy_upstream_to_thirdparty()