From 5beb604e21046823b3d2bbe5d1ba3137e36b4243 Mon Sep 17 00:00:00 2001 From: Rizo I Date: Sun, 20 Nov 2022 21:52:44 +0000 Subject: [PATCH] Support multiple opam repositories. --- .ocamlformat | 1 - TODO.md | 1 + bootstrap.nix | 4 +- nix/api.nix | 18 +++- nix/bootstrap.nix | 13 --- nix/overlays/default.nix | 7 ++ onix-lock.nix | 172 +++++++++++++++++----------------- onix/Lock_file.ml | 15 +-- onix/Lock_file.mli | 4 +- onix/Main.ml | 32 +++---- onix/Nix_utils.ml | 55 +++++++++++ onix/Nix_utils.mli | 2 + onix/Pp_lock.ml | 43 ++++----- onix/Solver.ml | 29 +++--- onix/Solver.mli | 4 +- onix/Utils.ml | 5 + shell.nix | 11 +-- tests/Test_lock.ml | 16 +--- tests/multiple-opam-repos.nix | 19 ++++ 19 files changed, 260 insertions(+), 191 deletions(-) delete mode 100644 nix/bootstrap.nix create mode 100644 tests/multiple-opam-repos.nix diff --git a/.ocamlformat b/.ocamlformat index 3bd5785..b461931 100644 --- a/.ocamlformat +++ b/.ocamlformat @@ -1,4 +1,3 @@ -version = 0.24.1 profile = conventional leading-nested-match-parens = false diff --git a/TODO.md b/TODO.md index 8ef959e..32cafa6 100644 --- a/TODO.md +++ b/TODO.md @@ -112,6 +112,7 @@ - Support easy building of any package without nix lock. - Support offline mode. - Prefetch packages to compute sha256 when md5 is used. + - Support multiple opam repositories. - [ ] Help debug nix paths (eg show opam repo path) - [x] When the scope is built but there was a pkg override do we still build the pkg? Must not. - [x] Replace hardcoded ocaml nix versions with a call to get currently available version from ocaml-ng. diff --git a/bootstrap.nix b/bootstrap.nix index 93cc764..11ac0c8 100644 --- a/bootstrap.nix +++ b/bootstrap.nix @@ -2,9 +2,9 @@ let ocamlPackages = pkgs.ocaml-ng.ocamlPackages_4_14; - onix = import ../default.nix { inherit pkgs; }; + onix = import ./default.nix { inherit pkgs; }; in { - scope = onix.build { lockFile = ../onix-lock.nix; }; + scope = onix.build { lockFile = ./onix-lock.nix; }; lock = onix.lock { repoUrl = "https://github.com/ocaml/opam-repository.git#f3dcd527e82e83facb92cd2727651938cb9fecf9"; diff --git a/nix/api.nix b/nix/api.nix index 06a43da..7f3d914 100644 --- a/nix/api.nix +++ b/nix/api.nix @@ -1,7 +1,7 @@ { pkgs ? import { }, onix }: let - defaultRepoUrl = "https://github.com/ocaml/opam-repository.git"; + defaultRepos = [ "https://github.com/ocaml/opam-repository.git" ]; defaultLockFile = "onix-lock.json"; defaultLogLevel = "debug"; defaultOverlay = import ./overlays/default.nix pkgs; @@ -283,6 +283,12 @@ let self: super: defaultOverlay self super // overrides self super; in scope.overrideScope' overlay; + joinRepositories = repositories: + pkgs.symlinkJoin { + name = "onix-opam-repo"; + paths = map builtins.fetchGit repositories; + }; + in rec { private = { }; @@ -291,24 +297,26 @@ in rec { , withDoc ? false, withDevSetup ? false }: let onixLock = lib.importJSON lockFile; - repoPath = builtins.fetchGit onixLock.repository; + repositories = onixLock.repositories; + repoPath = joinRepositories repositories; deps = onixLock.packages; scope = buildScope { inherit projectRoot repoPath withTest withDoc withDevSetup; } deps; in applyOverrides scope overrides; - lock = { repoUrl ? defaultRepoUrl, resolutions ? null + lock = { repositories ? defaultRepos, resolutions ? null , lockFile ? defaultLockFile, logLevel ? defaultLogLevel, withTest ? false , withDoc ? false, withDevSetup ? false, opamFiles ? [ ] }: let + repositoriesStr = lib.strings.concatStringsSep "," repositories; opamFilesStr = lib.strings.concatStrings (map (f: " " + builtins.toString f) opamFiles); in pkgs.mkShell { buildInputs = [ onix ]; shellHook = if isNull resolutions then '' onix lock \ - --repo-url='${repoUrl}' \ + --repositories='${repositoriesStr}' \ --lock-file='${builtins.toString lockFile}' \ --with-test=${builtins.toJSON withTest} \ --with-doc=${builtins.toJSON withDoc} \ @@ -317,7 +325,7 @@ in rec { exit $? '' else '' onix lock \ - --repo-url='${repoUrl}' \ + --repositories='${repositoriesStr}' \ --resolutions='${mkResolutionsArg resolutions}' \ --lock-file='${builtins.toString lockFile}' \ --with-test=${builtins.toJSON withTest} \ diff --git a/nix/bootstrap.nix b/nix/bootstrap.nix deleted file mode 100644 index 1e1098c..0000000 --- a/nix/bootstrap.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ pkgs ? import { } }: - -let - ocamlPackages = pkgs.ocaml-ng.ocamlPackages_4_14; - onix = import ../default.nix { inherit pkgs; }; -in { - scope = onix.build { lockFile = ./onix-lock.nix; }; - lock = onix.lock { - repoUrl = - "https://github.com/ocaml/opam-repository.git#f3dcd527e82e83facb92cd2727651938cb9fecf9"; - resolutions = { "ocaml-system" = "*"; }; - }; -} diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index 2808933..3e7e272 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -56,6 +56,13 @@ let # ''; # }); + num = super.num.overrideAttrs (selfAttrs: superAttrs: { + postInstall = '' + mkdir -p "$out/lib/ocaml/4.14.0/site-lib/stublibs" + mv $out/lib/ocaml/4.14.0/site-lib/num/*.so "$out/lib/ocaml/4.14.0/site-lib/stublibs" + ''; + }); + }; darwin = { diff --git a/onix-lock.nix b/onix-lock.nix index 07d0d7a..01a1903 100644 --- a/onix-lock.nix +++ b/onix-lock.nix @@ -4,7 +4,7 @@ let repo = builtins.fetchGit { rev = "f3dcd527e82e83facb92cd2727651938cb9fecf9"; }; in rec { - _0install-solver = { + "0install-solver" = { name = "0install-solver"; version = "2.18"; src = pkgs.fetchurl { @@ -12,10 +12,10 @@ rec { sha256 = "648c4b318c1a26dfcb44065c226ab8ca723795924ad80a3bf39ae1ce0e9920c3"; }; opam = "${repo}/packages/0install-solver/0install-solver.2.18/opam"; - depends = [ dune ocaml ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" ]; + buildDepends = [ "dune" ]; }; - astring = { + "astring" = { name = "astring"; version = "0.8.5"; src = pkgs.fetchurl { @@ -23,25 +23,25 @@ rec { sha256 = "1ykhg9gd3iy7zsgyiy2p9b1wkpqg9irw5pvcqs3sphq71iir4ml6"; }; opam = "${repo}/packages/astring/astring.0.8.5/opam"; - depends = [ ocaml ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "ocaml" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - base-bigarray = { + "base-bigarray" = { name = "base-bigarray"; version = "base"; opam = "${repo}/packages/base-bigarray/base-bigarray.base/opam"; }; - base-threads = { + "base-threads" = { name = "base-threads"; version = "base"; opam = "${repo}/packages/base-threads/base-threads.base/opam"; }; - base-unix = { + "base-unix" = { name = "base-unix"; version = "base"; opam = "${repo}/packages/base-unix/base-unix.base/opam"; }; - bos = { + "bos" = { name = "bos"; version = "0.2.1"; src = pkgs.fetchurl { @@ -49,10 +49,10 @@ rec { sha512 = "8daeb8a4c2dd1f2460f6274ada19f4f1b6ebe875ff83a938c93418ce0e6bdb74b8afc5c9a7d410c1c9df2dad030e4fa276b6ed2da580639484e8b5bc92610b1d"; }; opam = "${repo}/packages/bos/bos.0.2.1/opam"; - depends = [ astring base-unix fmt fpath logs ocaml rresult ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "astring" "base-unix" "fmt" "fpath" "logs" "ocaml" "rresult" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - cmdliner = { + "cmdliner" = { name = "cmdliner"; version = "1.1.1"; src = pkgs.fetchurl { @@ -60,9 +60,9 @@ rec { sha512 = "5478ad833da254b5587b3746e3a8493e66e867a081ac0f653a901cc8a7d944f66e4387592215ce25d939be76f281c4785702f54d4a74b1700bc8838a62255c9e"; }; opam = "${repo}/packages/cmdliner/cmdliner.1.1.1/opam"; - depends = [ ocaml ]; + depends = [ "ocaml" ]; }; - cppo = { + "cppo" = { name = "cppo"; version = "1.6.9"; src = pkgs.fetchurl { @@ -70,10 +70,10 @@ rec { sha512 = "26ff5a7b7f38c460661974b23ca190f0feae3a99f1974e0fd12ccf08745bd7d91b7bc168c70a5385b837bfff9530e0e4e41cf269f23dd8cf16ca658008244b44"; }; opam = "${repo}/packages/cppo/cppo.1.6.9/opam"; - depends = [ base-unix dune ocaml ]; - buildDepends = [ dune ]; + depends = [ "base-unix" "dune" "ocaml" ]; + buildDepends = [ "dune" ]; }; - dune = { + "dune" = { name = "dune"; version = "3.4.1"; src = pkgs.fetchurl { @@ -81,9 +81,9 @@ rec { sha256 = "299fa33cffc108cc26ff59d5fc9d09f6cb0ab3ac280bf23a0114cfdc0b40c6c5"; }; opam = "${repo}/packages/dune/dune.3.4.1/opam"; - depends = [ base-threads base-unix ocaml ]; + depends = [ "base-threads" "base-unix" "ocaml" ]; }; - easy-format = { + "easy-format" = { name = "easy-format"; version = "1.3.2"; src = pkgs.fetchurl { @@ -91,10 +91,10 @@ rec { sha256 = "3440c2b882d537ae5e9011eb06abb53f5667e651ea4bb3b460ea8230fa8c1926"; }; opam = "${repo}/packages/easy-format/easy-format.1.3.2/opam"; - depends = [ dune ocaml ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" ]; + buildDepends = [ "dune" ]; }; - fmt = { + "fmt" = { name = "fmt"; version = "0.9.0"; src = pkgs.fetchurl { @@ -102,10 +102,10 @@ rec { sha512 = "66cf4b8bb92232a091dfda5e94d1c178486a358cdc34b1eec516d48ea5acb6209c0dfcb416f0c516c50ddbddb3c94549a45e4a6d5c5fd1c81d3374dec823a83b"; }; opam = "${repo}/packages/fmt/fmt.0.9.0/opam"; - depends = [ base-unix cmdliner ocaml ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "base-unix" "cmdliner" "ocaml" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - fpath = { + "fpath" = { name = "fpath"; version = "0.7.3"; src = pkgs.fetchurl { @@ -113,10 +113,10 @@ rec { sha256 = "03z7mj0sqdz465rc4drj1gr88l9q3nfs374yssvdjdyhjbqqzc0j"; }; opam = "${repo}/packages/fpath/fpath.0.7.3/opam"; - depends = [ astring ocaml ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "astring" "ocaml" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - logs = { + "logs" = { name = "logs"; version = "0.7.0"; src = pkgs.fetchurl { @@ -124,32 +124,32 @@ rec { sha256 = "1jnmd675wmsmdwyb5mx5b0ac66g4c6gpv5s4mrx2j6pb0wla1x46"; }; opam = "${repo}/packages/logs/logs.0.7.0/opam"; - depends = [ base-threads cmdliner fmt ocaml ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "base-threads" "cmdliner" "fmt" "ocaml" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - ocaml = { + "ocaml" = { name = "ocaml"; version = "4.14.0"; opam = "${repo}/packages/ocaml/ocaml.4.14.0/opam"; - depends = [ ocaml-config ocaml-system ]; - buildDepends = [ ocaml-system ]; + depends = [ "ocaml-config" "ocaml-system" ]; + buildDepends = [ "ocaml-system" ]; flags = ["conf" ]; }; - ocaml-config = { + "ocaml-config" = { name = "ocaml-config"; version = "2"; opam = "${repo}/packages/ocaml-config/ocaml-config.2/opam"; - depends = [ ocaml-system ]; - buildDepends = [ ocaml-system ]; + depends = [ "ocaml-system" ]; + buildDepends = [ "ocaml-system" ]; }; - ocaml-system = { + "ocaml-system" = { name = "ocaml-system"; version = "4.14.0"; opam = "${repo}/packages/ocaml-system/ocaml-system.4.14.0/opam"; - depexts = [ pkgs.ocaml-ng.ocamlPackages_4_14.ocaml ]; + depexts = [ "ocaml-ng.ocamlPackages_4_14.ocaml" ]; flags = ["compiler" ]; }; - ocamlbuild = { + "ocamlbuild" = { name = "ocamlbuild"; version = "0.14.1"; src = pkgs.fetchurl { @@ -157,9 +157,9 @@ rec { sha512 = "1f5b43215b1d3dc427b9c64e005add9d423ed4bca9686d52c55912df8955647cb2d7d86622d44b41b14c4f0d657b770c27967c541c868eeb7c78e3bd35b827ad"; }; opam = "${repo}/packages/ocamlbuild/ocamlbuild.0.14.1/opam"; - depends = [ ocaml ]; + depends = [ "ocaml" ]; }; - ocamlfind = { + "ocamlfind" = { name = "ocamlfind"; version = "1.9.5"; src = pkgs.fetchurl { @@ -167,9 +167,9 @@ rec { sha512 = "03514c618a16b02889db997c6c4789b3436b3ad7d974348d2c6dea53eb78898ab285ce5f10297c074bab4fd2c82931a8b7c5c113b994447a44abb30fca74c715"; }; opam = "${repo}/packages/ocamlfind/ocamlfind.1.9.5/opam"; - depends = [ ocaml ]; + depends = [ "ocaml" ]; }; - ocamlgraph = { + "ocamlgraph" = { name = "ocamlgraph"; version = "2.0.0"; src = pkgs.fetchurl { @@ -177,19 +177,19 @@ rec { sha256 = "20fe267797de5322088a4dfb52389b2ea051787952a8a4f6ed70fcb697482609"; }; opam = "${repo}/packages/ocamlgraph/ocamlgraph.2.0.0/opam"; - depends = [ dune ocaml stdlib-shims ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" "stdlib-shims" ]; + buildDepends = [ "dune" ]; }; - onix = { + "onix" = { name = "onix"; version = "root"; src = pkgs.nix-gitignore.gitignoreSource [] ./.; opam = "${onix.src}/onix.opam"; - depends = [ bos cmdliner easy-format fmt fpath logs ocaml opam-0install - yojson ]; - buildDepends = [ dune ]; + depends = [ "bos" "cmdliner" "easy-format" "fmt" "fpath" "logs" "ocaml" + "opam-0install" "yojson" ]; + buildDepends = [ "dune" ]; }; - opam-0install = { + "opam-0install" = { name = "opam-0install"; version = "0.4.3"; src = pkgs.fetchurl { @@ -197,11 +197,11 @@ rec { sha256 = "d59e0ebddda58f798ff50ebe213c83893b5a7c340c38c20950574d67e6145b8a"; }; opam = "${repo}/packages/opam-0install/opam-0install.0.4.3/opam"; - depends = [ _0install-solver cmdliner dune fmt ocaml opam-file-format - opam-state ]; - buildDepends = [ dune ]; + depends = [ "0install-solver" "cmdliner" "dune" "fmt" "ocaml" + "opam-file-format" "opam-state" ]; + buildDepends = [ "dune" ]; }; - opam-core = { + "opam-core" = { name = "opam-core"; version = "2.1.3"; src = pkgs.fetchurl { @@ -209,10 +209,10 @@ rec { sha512 = "040e4f58f93e962ff422617ce0d35ed45dd86921a9aac3505914c33dd942d0e5e5771e7e1774046504f9aa84f32bc4fbd6ac7720fbea862d48bf1ca29e02cefc"; }; opam = "${repo}/packages/opam-core/opam-core.2.1.3/opam"; - depends = [ base-bigarray base-unix dune ocaml ocamlgraph re ]; - buildDepends = [ cppo dune ]; + depends = [ "base-bigarray" "base-unix" "dune" "ocaml" "ocamlgraph" "re" ]; + buildDepends = [ "cppo" "dune" ]; }; - opam-file-format = { + "opam-file-format" = { name = "opam-file-format"; version = "2.1.4"; src = pkgs.fetchurl { @@ -220,10 +220,10 @@ rec { sha512 = "fb5e584080d65c5b5d04c7d2ac397b69a3fd077af3f51eb22967131be22583fea507390eb0d7e6f5c92035372a9e753adbfbc8bfd056d8fd4697c6f95dd8e0ad"; }; opam = "${repo}/packages/opam-file-format/opam-file-format.2.1.4/opam"; - depends = [ dune ocaml ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" ]; + buildDepends = [ "dune" ]; }; - opam-format = { + "opam-format" = { name = "opam-format"; version = "2.1.3"; src = pkgs.fetchurl { @@ -231,10 +231,10 @@ rec { sha512 = "040e4f58f93e962ff422617ce0d35ed45dd86921a9aac3505914c33dd942d0e5e5771e7e1774046504f9aa84f32bc4fbd6ac7720fbea862d48bf1ca29e02cefc"; }; opam = "${repo}/packages/opam-format/opam-format.2.1.3/opam"; - depends = [ dune ocaml opam-core opam-file-format re ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" "opam-core" "opam-file-format" "re" ]; + buildDepends = [ "dune" ]; }; - opam-repository = { + "opam-repository" = { name = "opam-repository"; version = "2.1.3"; src = pkgs.fetchurl { @@ -242,10 +242,10 @@ rec { sha512 = "040e4f58f93e962ff422617ce0d35ed45dd86921a9aac3505914c33dd942d0e5e5771e7e1774046504f9aa84f32bc4fbd6ac7720fbea862d48bf1ca29e02cefc"; }; opam = "${repo}/packages/opam-repository/opam-repository.2.1.3/opam"; - depends = [ dune ocaml opam-format ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" "opam-format" ]; + buildDepends = [ "dune" ]; }; - opam-state = { + "opam-state" = { name = "opam-state"; version = "2.1.3"; src = pkgs.fetchurl { @@ -253,10 +253,10 @@ rec { sha512 = "040e4f58f93e962ff422617ce0d35ed45dd86921a9aac3505914c33dd942d0e5e5771e7e1774046504f9aa84f32bc4fbd6ac7720fbea862d48bf1ca29e02cefc"; }; opam = "${repo}/packages/opam-state/opam-state.2.1.3/opam"; - depends = [ dune ocaml opam-repository ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" "opam-repository" ]; + buildDepends = [ "dune" ]; }; - re = { + "re" = { name = "re"; version = "1.10.4"; src = pkgs.fetchurl { @@ -264,10 +264,10 @@ rec { sha256 = "83eb3e4300aa9b1dc7820749010f4362ea83524742130524d78c20ce99ca747c"; }; opam = "${repo}/packages/re/re.1.10.4/opam"; - depends = [ dune ocaml seq ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" "seq" ]; + buildDepends = [ "dune" ]; }; - rresult = { + "rresult" = { name = "rresult"; version = "0.7.0"; src = pkgs.fetchurl { @@ -275,16 +275,16 @@ rec { sha512 = "f1bb631c986996388e9686d49d5ae4d8aaf14034f6865c62a88fb58c48ce19ad2eb785327d69ca27c032f835984e0bd2efd969b415438628a31f3e84ec4551d3"; }; opam = "${repo}/packages/rresult/rresult.0.7.0/opam"; - depends = [ ocaml ]; - buildDepends = [ ocamlbuild ocamlfind topkg ]; + depends = [ "ocaml" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" "topkg" ]; }; - seq = { + "seq" = { name = "seq"; version = "base"; opam = "${repo}/packages/seq/seq.base/opam"; - depends = [ ocaml ]; + depends = [ "ocaml" ]; }; - stdlib-shims = { + "stdlib-shims" = { name = "stdlib-shims"; version = "0.3.0"; src = pkgs.fetchurl { @@ -292,10 +292,10 @@ rec { sha256 = "babf72d3917b86f707885f0c5528e36c63fccb698f4b46cf2bab5c7ccdd6d84a"; }; opam = "${repo}/packages/stdlib-shims/stdlib-shims.0.3.0/opam"; - depends = [ dune ocaml ]; - buildDepends = [ dune ]; + depends = [ "dune" "ocaml" ]; + buildDepends = [ "dune" ]; }; - topkg = { + "topkg" = { name = "topkg"; version = "1.0.5"; src = pkgs.fetchurl { @@ -303,10 +303,10 @@ rec { sha512 = "9450e9139209aacd8ddb4ba18e4225770837e526a52a56d94fd5c9c4c9941e83e0e7102e2292b440104f4c338fabab47cdd6bb51d69b41cc92cc7a551e6fefab"; }; opam = "${repo}/packages/topkg/topkg.1.0.5/opam"; - depends = [ ocaml ocamlbuild ]; - buildDepends = [ ocamlbuild ocamlfind ]; + depends = [ "ocaml" "ocamlbuild" ]; + buildDepends = [ "ocamlbuild" "ocamlfind" ]; }; - yojson = { + "yojson" = { name = "yojson"; version = "2.0.2"; src = pkgs.fetchurl { @@ -314,7 +314,7 @@ rec { sha256 = "876bb6f38af73a84a29438a3da35e4857c60a14556a606525b148c6fdbe5461b"; }; opam = "${repo}/packages/yojson/yojson.2.0.2/opam"; - depends = [ dune ocaml seq ]; - buildDepends = [ cppo dune ]; + depends = [ "dune" "ocaml" "seq" ]; + buildDepends = [ "cppo" "dune" ]; }; } diff --git a/onix/Lock_file.ml b/onix/Lock_file.ml index af0141b..0272715 100644 --- a/onix/Lock_file.ml +++ b/onix/Lock_file.ml @@ -1,10 +1,13 @@ type t = { - repo : OpamUrl.t; + repos : OpamUrl.t list; packages : Lock_pkg.t list; } -let make ~repo_url packages = - if Option.is_none repo_url.OpamUrl.hash then - Fmt.failwith "Repo URI without rev when creating a lock file: %a" - Opam_utils.pp_url repo_url; - { repo = repo_url; packages } +let make ~repos packages = + List.iter + (fun url -> + if Option.is_none url.OpamUrl.hash then + Fmt.failwith "Repo URI without rev when creating a lock file: %a" + Opam_utils.pp_url url) + repos; + { repos; packages } diff --git a/onix/Lock_file.mli b/onix/Lock_file.mli index 5971dec..62d1daa 100644 --- a/onix/Lock_file.mli +++ b/onix/Lock_file.mli @@ -1,6 +1,6 @@ type t = { - repo : OpamUrl.t; + repos : OpamUrl.t list; packages : Lock_pkg.t list; } -val make : repo_url:OpamUrl.t -> Lock_pkg.t list -> t +val make : repos:OpamUrl.t list -> Lock_pkg.t list -> t diff --git a/onix/Main.ml b/onix/Main.ml index e71c805..ae6749e 100644 --- a/onix/Main.ml +++ b/onix/Main.ml @@ -86,16 +86,16 @@ let with_dev_setup_arg ~absent = |> Arg.opt ~vopt:`root (Arg.enum flag_scopes) absent |> Arg.value -let repo_url_arg = +let repositories_arg = let doc = - "The URL of the OPAM repository to be used when solving the dependencies. \ - Use the following format: \ + "Comma-separated URLs of the OPAM repositories to be used when solving the \ + dependencies. Use the following format: \ https://github.com/ocaml/opam-repository.git[#HASH]" in - let docv = "URL" in + let docv = "LIST" in Arg.( - info ["repo-url"] ~env:(Cmd.Env.info "ONIX_REPO_URL") ~docv ~doc - |> opt string "https://github.com/ocaml/opam-repository.git" + info ["repositories"] ~docv ~doc + |> opt (list string) ["https://github.com/ocaml/opam-repository.git"] |> value) let mk_pkg_ctx ~ocaml_version ~opamfile ~prefix ~opam_pkg () = @@ -200,14 +200,14 @@ module Lock = struct let input_opam_files_arg = Arg.(value & pos_all file [] & info [] ~docv:"OPAM_FILE") - let run style_renderer log_level lock_file_path repo_url resolutions with_test + let run style_renderer log_level lock_file_path repos resolutions with_test with_doc with_dev_setup input_opam_files = setup_logs style_renderer log_level; - Logs.info (fun log -> log "lock: Running... repo_url=%S" repo_url); + Logs.info (fun log -> log "lock: Running..."); let lock_file = - Onix.Solver.solve ~repo_url ~resolutions ~with_test ~with_doc - ~with_dev_setup input_opam_files + Onix.Solver.solve ~repos ~resolutions ~with_test ~with_doc ~with_dev_setup + input_opam_files in Onix.Utils.Out_channel.with_open_text lock_file_path (fun chan -> let out = Format.formatter_of_out_channel chan in @@ -223,7 +223,7 @@ module Lock = struct $ Fmt_cli.style_renderer () $ Logs_cli.level ~env:(Cmd.Env.info "ONIX_LOG_LEVEL") () $ lock_file_arg - $ repo_url_arg + $ repositories_arg $ resolutions_arg $ with_test_arg ~absent:`root $ with_doc_arg ~absent:`root @@ -240,10 +240,10 @@ module Gen = struct let docv = "DIR" in Arg.(info ["lock-dir"] ~docv ~doc |> opt file "./onix.lock" |> value) - let run style_renderer log_level ignore_file lock_dir repo_url resolutions + let run style_renderer log_level ignore_file lock_dir repos resolutions with_test with_doc with_dev_setup input_opam_files = setup_logs style_renderer log_level; - Logs.info (fun log -> log "lock: Running... repo_url=%S" repo_url); + Logs.info (fun log -> log "gen: Running..."); let ignore_file = if String.equal ignore_file "none" then None @@ -260,8 +260,8 @@ module Gen = struct in let lock_file = - Onix.Solver.solve ~repo_url ~resolutions ~with_test ~with_doc - ~with_dev_setup input_opam_files + Onix.Solver.solve ~repos ~resolutions ~with_test ~with_doc ~with_dev_setup + input_opam_files in Onix.Gen_drv_tree.gen ~ignore_file ~lock_dir ~with_test ~with_doc ~with_dev_setup lock_file; @@ -277,7 +277,7 @@ module Gen = struct $ Logs_cli.level ~env:(Cmd.Env.info "ONIX_LOG_LEVEL") () $ ignore_file_arg $ lock_dir_arg - $ repo_url_arg + $ repositories_arg $ resolutions_arg $ with_test_arg ~absent:`root $ with_doc_arg ~absent:`root diff --git a/onix/Nix_utils.ml b/onix/Nix_utils.ml index 9530f30..630a00b 100644 --- a/onix/Nix_utils.ml +++ b/onix/Nix_utils.ml @@ -137,6 +137,61 @@ let prefetch_git_with_path url = in (rev, path) +let fetch_resolve_many_expr urls = + let url_to_nix (url : OpamUrl.t) = + match url.hash with + | Some hash -> + let url' = { url with OpamUrl.hash = None } in + Fmt.str "{ url = \"%a\"; rev = \"%s\"; }" Opam_utils.pp_url url' hash + | None -> Fmt.str "{ url = \"%a\"; }" Opam_utils.pp_url url + in + let urls = urls |> List.map url_to_nix |> String.concat " " in + Fmt.str + {| +let + urls = [ %s ]; + fetched = map (x: (builtins.fetchGit x) // { inherit (x) url; }) urls; + resolved = map (x: "${x.url}#${x.rev},${x.outPath}") fetched; +in + builtins.concatStringsSep ";" resolved +|} + urls + +let fetch_resolve_many urls = + let result = urls |> fetch_resolve_many_expr |> eval in + let lines = String.split_on_char ';' result in + List.map + (fun line -> + match String.split_on_char ',' line with + | [url; path] -> (OpamUrl.of_string url, OpamFilename.Dir.of_string path) + | _ -> Fmt.failwith "Invalid repo format: %s" line) + lines + +let symlink_join_expr ~name paths = + Fmt.str + {| +let pkgs = import {}; +in pkgs.symlinkJoin { + name = %S; + paths = [ %a ]; +} +|} + name + Fmt.(list ~sep:Fmt.sp Opam_utils.pp_filename_dir) + paths + +let symlink_join ~name paths = + let open Bos in + let expr = symlink_join_expr ~name paths in + let cmd = Cmd.(v "nix-build" % "--no-out-link" % "-E" % expr) in + let result = + cmd + |> OS.Cmd.run_out ~err:OS.Cmd.err_null + |> OS.Cmd.to_string + |> Utils.Result.force_with_msg + in + OpamFilename.Dir.of_string result + type store_path = { hash : string; package_name : OpamPackage.Name.t; diff --git a/onix/Nix_utils.mli b/onix/Nix_utils.mli index a166914..b8d4923 100644 --- a/onix/Nix_utils.mli +++ b/onix/Nix_utils.mli @@ -1,6 +1,8 @@ val get_nix_build_jobs : unit -> string val fetch : OpamUrl.t -> OpamFilename.Dir.t val fetch_resolve : OpamUrl.t -> string * OpamFilename.Dir.t +val fetch_resolve_many : OpamUrl.t list -> (OpamUrl.t * OpamFilename.Dir.t) list +val symlink_join : name:string -> OpamFilename.Dir.t list -> OpamFilename.Dir.t val prefetch_url_with_path : ?hash_type:[< `sha256 | `sha512 > `sha256] -> diff --git a/onix/Pp_lock.ml b/onix/Pp_lock.ml index bbfce81..7d583ec 100644 --- a/onix/Pp_lock.ml +++ b/onix/Pp_lock.ml @@ -6,21 +6,6 @@ let pp_name_quoted formatter name = let name = OpamPackage.Name.to_string name in Fmt.Dump.string formatter name -let pp_version f version = - let version = OpamPackage.Version.to_string version in - (* We require that the version does NOT contain any '-' or '~' characters. - - Note that nix will replace '~' to '-' automatically. - The version is parsed with Nix_utils.parse_store_path by splitting bytes - '- ' to obtain the Pkg_ctx.package information. - This is fine because the version in the lock file is mostly informative. *) - let set_valid_char i = - match String.get version i with - | '-' | '~' -> '+' - | valid -> valid - in - let version = String.init (String.length version) set_valid_char in - Fmt.pf f "%S" version - let pp_hash f (kind, hash) = match kind with | `SHA256 -> Fmt.pf f "\"sha256\": %S" hash @@ -78,15 +63,21 @@ let pp_pkg ppf (t : Lock_pkg.t) = let pp_version f version = Fmt.pf f "\"version\": %S" version -let pp_repo_uri f repo_url = - match repo_url.OpamUrl.hash with - | Some rev -> - Fmt.pf f "@[\"repository\": {@ \"url\": %a,@ \"rev\": %S@]@,}" - (Fmt.quote Opam_utils.pp_url) - { repo_url with OpamUrl.hash = None } - rev - | None -> - Fmt.invalid_arg "Repo URI without fragment: %a" Opam_utils.pp_url repo_url +let pp_repos = + let pp_repo_url ppf repo_url = + match repo_url.OpamUrl.hash with + | None -> + Fmt.invalid_arg "Repo URI without fragment: %a" Opam_utils.pp_url repo_url + | Some rev -> + Fmt.pf ppf "@[{@ \"url\": %a,@ \"rev\": %S@]@,}" + (Fmt.quote Opam_utils.pp_url) + { repo_url with OpamUrl.hash = None } + rev + in + fun f repos -> + Fmt.pf f "@[\"repositories\": [@,%a@]@,]" + (Fmt.list ~sep:Fmt.comma pp_repo_url) + repos let pp_packages f deps = let pp_pkg fmt pkg = @@ -97,5 +88,5 @@ let pp_packages f deps = Fmt.pf f "@[\"packages\" : {@,%a@]@,}" (Fmt.hvbox pp_list) deps let pp fmt (t : Lock_file.t) = - Fmt.pf fmt {|{@[@,%a,@,%a,@,%a@]@,}@.|} pp_version Lib.version pp_repo_uri - t.repo pp_packages t.packages + Fmt.pf fmt {|{@[@,%a,@,%a,@,%a@]@,}@.|} pp_version Lib.version pp_repos + t.repos pp_packages t.packages diff --git a/onix/Solver.ml b/onix/Solver.ml index 6c1dce9..3d360b3 100644 --- a/onix/Solver.ml +++ b/onix/Solver.ml @@ -1,17 +1,22 @@ module Opam_0install_solver = Opam_0install.Solver.Make (Solver_context) -let resolve_repo repo_url = - let path, url = - let url = OpamUrl.of_string repo_url in - if Option.is_some url.hash then (Nix_utils.fetch url, url) - else - let rev, path = Nix_utils.fetch_resolve url in - (path, { url with hash = Some rev }) +let resolve_repos repos = + let repos = List.map OpamUrl.of_string repos in + let resolved_with_path = Nix_utils.fetch_resolve_many repos in + let joint_path = + match resolved_with_path with + | [(_repo_url, path)] -> path + | _ -> + Nix_utils.symlink_join ~name:"onix-opam-repo" + (List.map snd resolved_with_path) in - Logs.info (fun log -> log "Using OPAM repository: %a" Opam_utils.pp_url url); - (path, url) + let resolved_urls = List.map fst resolved_with_path in + Fmt.epr "@[Repositories:@,%a@]@." + Fmt.(list ~sep:cut (any "- " ++ Opam_utils.pp_url)) + resolved_urls; + (joint_path, resolved_urls) -let solve ?(resolutions = []) ~repo_url ~with_test ~with_doc ~with_dev_setup +let solve ?(resolutions = []) ~repos ~with_test ~with_doc ~with_dev_setup input_opam_files = let resolutions = Resolutions.make resolutions in Resolutions.debug resolutions; @@ -39,7 +44,7 @@ let solve ?(resolutions = []) ~repo_url ~with_test ~with_doc ~with_dev_setup (OpamPackage.Name.Map.keys root_opam_details) in - let repo_dir, repo_url = resolve_repo repo_url in + let repo_dir, resolved_repos = resolve_repos repos in let constraints = Resolutions.constraints resolutions in @@ -94,7 +99,7 @@ let solve ?(resolutions = []) ~repo_url ~with_test ~with_doc ~with_dev_setup None | some -> some) |> OpamPackage.Name.Map.values - |> Lock_file.make ~repo_url + |> Lock_file.make ~repos:resolved_repos | Error err -> prerr_endline (Opam_0install_solver.diagnostics err); exit 2 diff --git a/onix/Solver.mli b/onix/Solver.mli index 2f0cf2e..59aa91e 100644 --- a/onix/Solver.mli +++ b/onix/Solver.mli @@ -1,8 +1,6 @@ -val resolve_repo : string -> OpamFilename.Dir.t * OpamUrl.t - val solve : ?resolutions:OpamFormula.atom list -> - repo_url:string -> + repos:string list -> with_test:Opam_utils.dep_flag_scope -> with_doc:Opam_utils.dep_flag_scope -> with_dev_setup:Opam_utils.dep_flag_scope -> diff --git a/onix/Utils.ml b/onix/Utils.ml index e4a4700..3b64de7 100644 --- a/onix/Utils.ml +++ b/onix/Utils.ml @@ -38,6 +38,11 @@ module List = struct let is_not_empty = function | [] -> false | _ -> true + + let is_singleton t = + match t with + | [_] -> true + | _ -> false end module String = struct diff --git a/shell.nix b/shell.nix index f5af444..bcfb947 100644 --- a/shell.nix +++ b/shell.nix @@ -1,10 +1,7 @@ -{ pkgs ? import { } }: -let onix = import ./. { inherit pkgs; }; +{ pkgs ? import { }, ocamlPackages ? pkgs.ocaml-ng.ocamlPackages_4_14 +}: +let onix = import ./. { inherit pkgs ocamlPackages; }; in pkgs.mkShell { inputsFrom = [ onix ]; - buildInputs = [ - pkgs.nixfmt - pkgs.ocaml-ng.ocamlPackages_4_14.ocaml-lsp - pkgs.ocamlformat - ]; + buildInputs = [ pkgs.nixfmt ocamlPackages.ocaml-lsp pkgs.ocamlformat ]; } diff --git a/tests/Test_lock.ml b/tests/Test_lock.ml index 4f3f97e..174e776 100644 --- a/tests/Test_lock.ml +++ b/tests/Test_lock.ml @@ -97,9 +97,7 @@ let mk_lock ~name str = let test_complex_opam () = let lock_pkg = mk_lock ~name:"complex.root" complex_opam in - let actual = - Fmt.str "%a@." (Onix.Pp_lock_nix.pp_pkg ~ignore_file:None) lock_pkg - in + let actual = Fmt.str "%a@." Onix.Pp_lock.pp_pkg lock_pkg in let expected = {|name = "complex"; version = "root"; src = ./.; opam = "${src}/complex.opam"; depends = with self; [ bos cmdliner dune easy-format fmt fpath logs ocaml @@ -112,9 +110,7 @@ depexts = with pkgs; [ libogg ]; let test_dev_opam () = let lock_pkg = mk_lock ~name:"dev.dev" dev_opam in - let actual = - Fmt.str "%a@." (Onix.Pp_lock_nix.pp_pkg ~ignore_file:None) lock_pkg - in + let actual = Fmt.str "%a@." Onix.Pp_lock.pp_pkg lock_pkg in let expected = {|name = "dev"; version = "dev"; src = builtins.fetchGit { @@ -128,9 +124,7 @@ opam = "${src}/dev.opam"; let test_zip_src_opam () = let lock_pkg = mk_lock ~name:"zip.1.0.2" zip_src_opam in - let actual = - Fmt.str "%a@." (Onix.Pp_lock_nix.pp_pkg ~ignore_file:None) lock_pkg - in + let actual = Fmt.str "%a@." Onix.Pp_lock.pp_pkg lock_pkg in let expected = {|name = "zip"; version = "1.0.2"; src = pkgs.fetchurl { @@ -144,9 +138,7 @@ depexts = with pkgs; [ unzip ]; let test_other_deps_opam () = let lock_pkg = mk_lock ~name:"other-deps.1.0.1" other_deps_opam in - let actual = - Fmt.str "%a@." (Onix.Pp_lock_nix.pp_pkg ~ignore_file:None) lock_pkg - in + let actual = Fmt.str "%a@." Onix.Pp_lock.pp_pkg lock_pkg in let expected = {|name = "other-deps"; version = "1.0.1"; opam = "${repo}/packages/other-deps/other-deps.1.0.1/opam"; diff --git a/tests/multiple-opam-repos.nix b/tests/multiple-opam-repos.nix new file mode 100644 index 0000000..d9eef92 --- /dev/null +++ b/tests/multiple-opam-repos.nix @@ -0,0 +1,19 @@ +let + pkgs = import { }; + repos = [ + { url = "https://github.com/kit-ty-kate/opam-alpha-repository.git"; } + { + url = "https://github.com/ocaml/opam-repository.git"; + rev = "fe53d261c062c23d8271f6887702b9bc7459ad2e"; + } + ]; + paths = map (repo: (builtins.fetchGit repo) // { inherit (repo) url; }) repos; + path = pkgs.symlinkJoin { + name = "onix-opam-repos"; + inherit paths; + }; + json = builtins.toJSON { + repos = map (x: "${x.url}#${x.rev}") paths; + path = path; + }; +in path