Automatically update all Manifest files

.gitignore

@@ -2,5 +2,5 @@
 *.jl.cov
 *.jl.mem
 .DS_Store
-/Manifest.toml
+Manifest.toml
 /dev/

README.md

@@ -28,31 +28,38 @@ CompatHelper is now installed as a GitHub Action on your repository.
 
 
 ## Actions Setup
-* Sign up for the beta of GitHub Actions from https://github.com/features/actions 
+* Sign up for the beta of GitHub Actions from https://github.com/features/actions
 * Open the specific repository, navigate to the Settings tab, click Actions option, check if the Actions is enabled for this repository.
 
 
-## Updating `Manifest.toml` files after updating compat information
+## Custom pre-commit hooks
 
-If you check in `Manifest.toml` file(s), the PR created by default CompatHelper setup does not run CI with the latest versions of the dependencies.  In this case, it may be a good idea to run `Pkg.update()` via CompatHelper.  This can be done simply using the following snippet instead `run: julia -e 'using CompatHelper; CompatHelper.main()'`:
+CompatHelper supports running a custom function just before commiting changes.
+By default, this function updates any `Manifest.toml` files in your repository to reflect new compatibility bounds.
+If you want to extend this behaviour, you can pass your own zero-argument function to `CompatHelper.main`, like so:
 
 ```yaml
-run: >-
-  julia -e '
+run: julia -e '
   using CompatHelper;
   CompatHelper.main() do;
-      run(`julia --project=test -e "import Pkg; Pkg.instantiate(); Pkg.update()"`);
-      run(`julia --project=docs -e "import Pkg; Pkg.instantiate(); Pkg.update()"`);
-  end
-  '
+    CompatHelper.update_manifests();
+    println("I did it!");
+  end;'
 ```
 
-This setup updates `test/Manifest.toml` and `docs/Manifest.toml` before CompatHelper creates a commit for the pull request.
-
-This snippet uses `>-` to specify a long one-liner command using multi-line code (i.e., the shell does not see the newline characters after `;`).  Note that every line must ends with `;` when using `>-`.  Do not use `'` inside (outer) Julia code since it is used to quote the command line option `-e '...'`.
+This snippet uses `;` to specify the ends of lines, because according to YAML, the entire block of Julia code is a single line.
+Also to note is that you cannot use `'` inside of your Julia command, since it's used to quote the Julia code.
 
 A full example is available here: https://github.com/tkf/Kaleido.jl/blob/master/.github/workflows/CompatHelper.yml
 
+If you want to disable the default behavior, simply pass a dummy function that does nothing:
+
+```yaml
+run: julia -e '
+  using CompatHelper;
+  CompatHelper.main( () -> () );'
+```
+
 ## Acknowledgements
 
 - This work was supported in part by National Institutes of Health grants U54GM115677, R01LM011963, and R25MH116440. The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.

src/CompatHelper.jl

@@ -13,6 +13,7 @@ include("new_versions.jl")
 include("pull_requests.jl")
 include("stdlib.jl")
 include("timestamps.jl")
+include("update_manifests.jl")
 include("utils.jl")
 include("version_numbers.jl")
 

src/main.jl

@@ -4,15 +4,12 @@ const default_registries = Pkg.Types.RegistrySpec[Pkg.RegistrySpec(name = "Gener
                                                                    uuid = "23338594-aafe-5451-b93e-139f81909106",
                                                                    url = "https://github.com/JuliaRegistries/General.git")]
 
-const default_value_for_keep_existing_compat = true
-const default_value_for_drop_existing_compat = false
-
-function main(precommit_hook::Function = () -> (),
+function main(precommit_hook::Function = update_manifests,
               env::AbstractDict = ENV,
               ci_cfg::CIService = auto_detect_ci_service(; env = env);
               registries::Vector{Pkg.Types.RegistrySpec} = default_registries,
-              keep_existing_compat::Bool = default_value_for_keep_existing_compat,
-              drop_existing_compat::Bool = default_value_for_drop_existing_compat,
+              keep_existing_compat::Bool = true,
+              drop_existing_compat::Bool = false,
               master_branch::Union{DefaultBranch, AbstractString} = DefaultBranch(),
               pr_title_prefix::String = "")
     if !keep_existing_compat && !drop_existing_compat

src/update_manifests.jl

@@ -0,0 +1,75 @@
+function update_manifests(path::AbstractString = pwd();
+                          registries::Vector{Pkg.Types.RegistrySpec} = default_registries,
+                          delete_old_manifest::Bool = true)
+    environments_to_update = Vector{String}(undef, 0)
+    for (root, dirs, files) in walkdir(path)
+        for file in files
+            if (file == "Manifest.toml") | (file == "JuliaManifest.toml")
+                environment = joinpath(path, root)
+                push!(environments_to_update, environment)
+            end
+        end
+    end
+    unique!(environments_to_update)
+    _update_manifests(environments_to_update;
+                      registries = registries,
+                      delete_old_manifest = delete_old_manifest)
+    return nothing
+end
+
+function _update_manifests(environments::AbstractVector{<:AbstractString};
+                           registries::Vector{Pkg.Types.RegistrySpec},
+                           delete_old_manifest::Bool)
+    for environment in environments
+        _update_manifest(environment;
+                         registries = registries,
+                         delete_old_manifest = delete_old_manifest)
+    end
+    return nothing
+end
+
+function _has_project(environment::AbstractString)
+    return (ispath(joinpath(environment, "Project.toml"))) |
+           (ispath(joinpath(environment, "JuliaProject.toml")))
+end
+
+function _update_manifest(environment::AbstractString;
+                          registries::Vector{Pkg.Types.RegistrySpec},
+                          delete_old_manifest::Bool)
+    @info("Updating environment: $(environment)")
+    if _has_project(environment) && delete_old_manifest
+        @debug("Removing Manifest.toml if it exists")
+        rm(joinpath(environment, "Manifest.toml");
+           force = true, recursive = true)
+        @debug("Removing JuliaManifest.toml if it exists")
+        rm(joinpath(environment, "JuliaManifest.toml");
+           force = true, recursive = true)
+    end
+    with_tmp_dir() do tmp_dir
+        code = """
+            using Pkg;
+            using UUIDs;
+            Pkg.Types.RegistrySpec(name::Union{Nothing, String},
+                                   uuid::Union{Nothing, UUID},
+                                   url::Union{Nothing, String},
+                                   path::Union{Nothing, String}) = Pkg.Types.RegistrySpec(;
+                                                                                          name = name,
+                                                                                          uuid = uuid,
+                                                                                          url = url,
+                                                                                          path = path)
+            registries = $(registries);
+            for registry in registries;
+                Pkg.Registry.add(registry);
+            end;
+            Pkg.activate("$(environment)"; shared = false);
+            Pkg.instantiate();
+            Pkg.update();
+            """
+        cmd = Cmd(`$(Base.julia_cmd()) -e $(code)`;
+                  env = Dict("PATH" => ENV["PATH"],
+                             "JULIA_DEPOT_PATH" => tmp_dir))
+        run(pipeline(cmd, stdout=stdout, stderr=stderr))
+        return nothing
+    end
+    return nothing
+end

src/utils.jl

@@ -16,3 +16,11 @@ function generate_pr_title_parenthetical(keep_or_drop::Symbol,
         return ""
     end
 end
+
+function with_tmp_dir(f::Function)
+    tmp_dir = mktempdir()
+    atexit(() -> rm(tmp_dir; force = true, recursive = true))
+    result = f(tmp_dir)
+    rm(tmp_dir; force = true, recursive = true)
+    return result
+end

test/integration-tests.jl

@@ -81,6 +81,20 @@ with_master_branch(templates("master_4"), "master"; repo_url = repo_url_with_aut
     end
 end
 
+with_master_branch(templates("master_5"), "master"; repo_url = repo_url_with_auth) do master_5
+    withenv("GITHUB_REPOSITORY" => COMPATHELPER_INTEGRATION_TEST_REPO,
+            "GITHUB_TOKEN" => TEST_USER_GITHUB_TOKEN) do
+        precommit_hook = CompatHelper.update_manifests
+        env = ENV
+        ci_cfg = CompatHelper.GitHubActions(whoami)
+        CompatHelper.main(precommit_hook, env, ci_cfg;
+                          pr_title_prefix = "[test-5a] ",
+                          master_branch = master_5,
+                          keep_existing_compat = false,
+                          drop_existing_compat = true)
+    end
+end
+
 all_prs = CompatHelper.get_all_pull_requests(repo, "open";
                                              auth = auth,
                                              per_page = 3,

test/templates/master_5/.gitignore

@@ -0,0 +1,2 @@
+!Manifest.toml
+!/Manifest.toml

test/templates/master_5/Manifest.toml

@@ -0,0 +1,189 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[ArgCheck]]
+deps = ["Random"]
+git-tree-sha1 = "dab25d711a1dedb707a55dbc1eb9fd578f76ff32"
+uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197"
+version = "1.0.1"
+
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[CSTParser]]
+deps = ["Tokenize"]
+git-tree-sha1 = "99dda94f5af21a4565dc2b97edf6a95485f116c3"
+uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
+version = "1.0.0"
+
+[[Compat]]
+deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
+git-tree-sha1 = "ed2c4abadf84c53d9e58510b5fc48912c2336fbb"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "2.2.0"
+
+[[Crayons]]
+deps = ["Test"]
+git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.0.0"
+
+[[DataAPI]]
+git-tree-sha1 = "674b67f344687a88310213ddfa8a2b3c76cc4252"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.1.0"
+
+[[DataStructures]]
+deps = ["InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "1fe8fad5fc84686dcbc674aa255bc867a64f8132"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.17.5"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[DefaultApplication]]
+deps = ["Test"]
+git-tree-sha1 = "a51d16b075dc52e22cde13b4a6e0ba4ba86649ee"
+uuid = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85"
+version = "0.1.3"
+
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[DocStringExtensions]]
+deps = ["LibGit2", "Markdown", "Pkg", "Test"]
+git-tree-sha1 = "88bb0edb352b16608036faadcc071adda068582a"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.8.1"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[LibGit2]]
+deps = ["Printf"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[MacroTools]]
+deps = ["CSTParser", "Compat", "DataStructures", "Test", "Tokenize"]
+git-tree-sha1 = "d6e9dedb8c92c3465575442da456aec15a89ff76"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.1"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[Missings]]
+deps = ["DataAPI"]
+git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5"
+uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
+version = "0.4.3"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[OrderedCollections]]
+deps = ["Random", "Serialization", "Test"]
+git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.1.0"
+
+[[PGFPlotsX]]
+deps = ["ArgCheck", "Crayons", "DataStructures", "Dates", "DefaultApplication", "DocStringExtensions", "MacroTools", "Missings", "Parameters", "Requires", "StatsBase", "Unicode"]
+git-tree-sha1 = "b75f57b00e24934ad94594a71bb10e8c3c523064"
+uuid = "8314cec4-20b6-5062-9cdb-752b83310925"
+version = "1.0.0"
+
+[[Parameters]]
+deps = ["OrderedCollections"]
+git-tree-sha1 = "b62b2558efb1eef1fa44e4be5ff58a515c287e38"
+uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a"
+version = "0.12.0"
+
+[[Pkg]]
+deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[Requires]]
+deps = ["Test"]
+git-tree-sha1 = "f6fbf4ba64d295e146e49e021207993b6b48c7d1"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "0.5.2"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[SharedArrays]]
+deps = ["Distributed", "Mmap", "Random", "Serialization"]
+uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[SortingAlgorithms]]
+deps = ["DataStructures", "Random", "Test"]
+git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd"
+uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
+version = "0.3.1"
+
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[[StatsBase]]
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"]
+git-tree-sha1 = "c53e809e63fe5cf5de13632090bc3520649c9950"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.32.0"
+
+[[Test]]
+deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[Tokenize]]
+git-tree-sha1 = "dfcdbbfb2d0370716c815cbd6f8a364efb6f42cf"
+uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
+version = "0.5.6"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"