Skip to content

Commit

Permalink
Use JSON for lock format.
Browse files Browse the repository at this point in the history
  • Loading branch information
rizo committed Nov 20, 2022
1 parent ffe36f0 commit 983425a
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 224 deletions.
93 changes: 60 additions & 33 deletions nix/api.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

let
defaultRepoUrl = "https://github.com/ocaml/opam-repository.git";
defaultLockFile = "./onix-lock.nix";
defaultLockFile = "onix-lock.json";
defaultLogLevel = "debug";
defaultOverlay = import ./overlays/default.nix pkgs;

Expand Down Expand Up @@ -37,8 +37,7 @@ let
else
throw "invalid dependency flag value: ${depFlag}";

fetchSrc = { projectRoot, gitignore ? null }:
name: src:
fetchSrc = { projectRoot, name, src }:
let urlLen = builtins.stringLength src.url;
in if lib.strings.hasPrefix "file://" src.url then
# local url
Expand All @@ -48,12 +47,10 @@ let
projectRoot
else
"${projectRoot}/path";
in (if isNull gitignore then
projectPath
else if gitignore == ".gitignore" then
in (if pathExists "${projectPath}/.gitignore" then
pkgs.nix-gitignore.gitignoreSource [ ] projectPath
else
pkgs.nix-gitignore.gitignoreSourcePure [ gitignore ] projectPath)
projectPath)
else if lib.strings.hasPrefix "git+" src.url then
# git url
builtins.fetchGit {
Expand All @@ -67,9 +64,30 @@ let
else
throw "invalid src for package ${name}: ${builtins.toJSON src}";

getOpamFile = { repoPath, src, name, version }:
if version == "root" || version == "dev" then
if pathExists "${src}/${name}.opam" then
"${src}/${name}.opam"
else if pathExists "${src}/opam" then
"${src}/opam"
else
throw "could not find opam file for package ${name} in ${src}"
else if version == "dev" then
"${src}/${name}.opam"
else
"${repoPath}/packages/${name}/${name}.${version}/opam";

# We require that the version does NOT contain any '-' or '~' characters.
# - Note that nix will replace '~' to '-' automatically.
# The version is parsed with Nix_utils.parse_store_path by splitting bytes
# '- ' to obtain the Pkg_ctx.package information.
# This is fine because the version in the lock file is mostly informative.
normalizeVersion = version:
builtins.replaceStrings [ "-" "~" ] [ "+" "+" ] version;

# Build a package from a lock dependency.
buildPkg = { projectRoot, scope, withTest, withDoc, withDevSetup }:
dep:
buildPkg = { projectRoot, repoPath, scope, withTest, withDoc, withDevSetup }:
name: dep:
let
ocaml = scope.ocaml;

Expand All @@ -86,11 +104,17 @@ let
(dep.depexts or [ ]);

src = if dep ? src then
fetchSrc { inherit projectRoot; } dep.name dep.src
fetchSrc {
inherit projectRoot name;
inherit (dep) src;
}
else
null;

flags = dep.flags or [ ];
opam = getOpamFile {
inherit repoPath src name;
inherit (dep) version;
};

onixPathHook = pkgs.makeSetupHook { name = "onix-path-hook"; }
(pkgs.writeText "onix-path-hook.sh" ''
Expand All @@ -114,8 +138,8 @@ let

in stdenv.mkDerivation {
inherit src;
pname = dep.name;
version = dep.version;
pname = name;
version = normalizeVersion dep.version;
dontUnpack = isNull src;
strictDeps = true;
dontStrip = false;
Expand All @@ -135,9 +159,9 @@ let
prePatch = ''
${onix}/bin/onix opam-patch \
--ocaml-version=${ocaml.version} \
--opam=${dep.opam} \
--opam=${opam} \
--path=$out \
${dep.name}.${dep.version}
${name}.${dep.version}
'';

# Steps:
Expand Down Expand Up @@ -167,12 +191,12 @@ let
${onix}/bin/onix opam-build \
--ocaml-version=${ocaml.version} \
--opam=${dep.opam} \
--opam=${opam} \
--with-test=${builtins.toJSON withTest} \
--with-doc=${builtins.toJSON withDoc} \
--with-dev-setup=${builtins.toJSON withDevSetup} \
--path=$out \
${dep.name}.${dep.version}
${name}.${dep.version}
runHook postBuild
'';
Expand All @@ -196,24 +220,24 @@ let
-libdir="$out/lib/ocaml/${ocaml.version}/site-lib"
# .config files
if [[ -e "./${dep.name}.config" ]]; then
if [[ -e "./${name}.config" ]]; then
mkdir -p "$out/etc"
cp "./${dep.name}.config" "$out/etc/${dep.name}.config"
cp "./${name}.config" "$out/etc/${name}.config"
fi
mkdir -p "$OCAMLFIND_DESTDIR/${dep.name}"
mkdir -p "$OCAMLFIND_DESTDIR/${name}"
${onix}/bin/onix opam-install \
--ocaml-version=${ocaml.version} \
--opam=${dep.opam} \
--opam=${opam} \
--with-test=${builtins.toJSON withTest} \
--with-doc=${builtins.toJSON withDoc} \
--with-dev-setup=${builtins.toJSON withDevSetup} \
--path=$out \
${dep.name}.${dep.version}
${name}.${dep.version}
if [[ -e "$out/lib/${dep.name}/META" ]] && [[ ! -e "$OCAMLFIND_DESTDIR/${dep.name}" ]]; then
mv "$out/lib/${dep.name}" "$OCAMLFIND_DESTDIR"
if [[ -e "$out/lib/${name}/META" ]] && [[ ! -e "$OCAMLFIND_DESTDIR/${name}" ]]; then
mv "$out/lib/${name}" "$OCAMLFIND_DESTDIR"
fi
runHook postInstall
Expand All @@ -239,15 +263,15 @@ let
name + "=" + value) resolutions);

# Build a package scope from the locked deps.
buildScope = { projectRoot, withTest, withDoc, withDevSetup }:
buildScope = { projectRoot, repoPath, withTest, withDoc, withDevSetup }:
deps:
pkgs.lib.makeScope pkgs.newScope (self:
(mapAttrs' (_name: dep: {
name = dep.name;
(mapAttrs' (name: dep: {
inherit name;
value = buildPkg {
inherit projectRoot withTest withDoc withDevSetup;
inherit projectRoot repoPath withTest withDoc withDevSetup;
scope = self;
} dep;
} name dep;
}) deps));

# Apply default and user-provided overrides to the scope.
Expand All @@ -262,13 +286,16 @@ let
in rec {
private = { };

build = { projectRoot, lockFile ? projectRoot ++ "/onix-lock.nix"
build = { projectRoot, lockFile ? "${projectRoot}/${defaultLockFile}"
, overrides ? null, logLevel ? defaultLogLevel, withTest ? false
, withDoc ? false, withDevSetup ? false }:
let
deps = import lockFile { inherit pkgs; };
scope =
buildScope { inherit projectRoot withTest withDoc withDevSetup; } deps;
onixLock = lib.importJSON lockFile;
repoPath = builtins.fetchGit onixLock.repository;
deps = onixLock.packages;
scope = buildScope {
inherit projectRoot repoPath withTest withDoc withDevSetup;
} deps;
in applyOverrides scope overrides;

lock = { repoUrl ? defaultRepoUrl, resolutions ? null
Expand Down
2 changes: 1 addition & 1 deletion onix/Lib.ml
Original file line number Diff line number Diff line change
@@ -1 +1 @@
let version = "0.0.6"
let version = "0.0.7"
21 changes: 3 additions & 18 deletions onix/Main.ml
Original file line number Diff line number Diff line change
Expand Up @@ -200,32 +200,18 @@ module Lock = struct
let input_opam_files_arg =
Arg.(value & pos_all file [] & info [] ~docv:"OPAM_FILE")

let run style_renderer log_level ignore_file lock_file_path repo_url
resolutions with_test with_doc with_dev_setup input_opam_files =
let run style_renderer log_level lock_file_path repo_url resolutions with_test
with_doc with_dev_setup input_opam_files =
setup_logs style_renderer log_level;
Logs.info (fun log -> log "lock: Running... repo_url=%S" repo_url);

let ignore_file =
if String.equal ignore_file "none" then None
else if Sys.file_exists ignore_file then (
Logs.debug (fun log ->
log "Using %S ignore file to filter root sources." ignore_file);
Some ignore_file)
else (
Logs.warn (fun log ->
log
"The ignore file %S does not exist, will not filter root sources."
ignore_file);
None)
in

let lock_file =
Onix.Solver.solve ~repo_url ~resolutions ~with_test ~with_doc
~with_dev_setup input_opam_files
in
Onix.Utils.Out_channel.with_open_text lock_file_path (fun chan ->
let out = Format.formatter_of_out_channel chan in
Fmt.pf out "%a" (Onix.Pp_lock_nix.pp ~ignore_file) lock_file);
Fmt.pf out "%a" Onix.Pp_lock.pp lock_file);
Logs.info (fun log -> log "Created a lock file at %S." lock_file_path)

let info = Cmd.info "lock" ~doc:"Solve dependencies and create a lock file."
Expand All @@ -236,7 +222,6 @@ module Lock = struct
const run
$ Fmt_cli.style_renderer ()
$ Logs_cli.level ~env:(Cmd.Env.info "ONIX_LOG_LEVEL") ()
$ ignore_file_arg
$ lock_file_arg
$ repo_url_arg
$ resolutions_arg
Expand Down
101 changes: 101 additions & 0 deletions onix/Pp_lock.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
open Utils

(* Lock pkg printers *)

let pp_name_quoted formatter name =
let name = OpamPackage.Name.to_string name in
Fmt.Dump.string formatter name

let pp_version f version =
let version = OpamPackage.Version.to_string version in
(* We require that the version does NOT contain any '-' or '~' characters.
- Note that nix will replace '~' to '-' automatically.
The version is parsed with Nix_utils.parse_store_path by splitting bytes
'- ' to obtain the Pkg_ctx.package information.
This is fine because the version in the lock file is mostly informative. *)
let set_valid_char i =
match String.get version i with
| '-' | '~' -> '+'
| valid -> valid
in
let version = String.init (String.length version) set_valid_char in
Fmt.pf f "%S" version

let pp_hash f (kind, hash) =
match kind with
| `SHA256 -> Fmt.pf f "\"sha256\": %S" hash
| `SHA512 -> Fmt.pf f "\"sha512\": %S" hash
| `MD5 -> Fmt.pf f "\"md5\": %S" hash

let pp_src f (t : Lock_pkg.t) =
if Lock_pkg.is_root t then
let path =
let opam_path = t.opam_details.Opam_utils.path in
let path = OpamFilename.(Dir.to_string (dirname opam_path)) in
if String.equal path "./." || String.equal path "./" then "." else path
in
Fmt.pf f ",@,\"src\": { \"url\": \"file://%s\" }" path
else
match t.src with
| None -> ()
| Some (Git { url; rev }) ->
Fmt.pf f ",@,@[<v2>\"src\": {@,\"url\": \"git+%s\",@,\"rev\": %S@]@,}" url
rev
(* MD5 hashes are not supported by Nix fetchers. Fetch without hash.
This normally would not happen as we try to prefetch_src_if_md5. *)
| Some (Http { url; hash = `MD5, _ }) ->
Fmt.invalid_arg "Unexpected md5 hash: package=%a url=%a"
Opam_utils.pp_package t.opam_details.package Opam_utils.pp_url url
| Some (Http { url; hash }) ->
Fmt.pf f ",@,@[<v2>\"src\": {@,\"url\": %a,@,%a@]@,}"
(Fmt.quote Opam_utils.pp_url)
url pp_hash hash

let pp_depends =
let pp_deps = Fmt.iter ~sep:Fmt.comma Name_set.iter pp_name_quoted in
fun key f deps ->
if Name_set.is_empty deps then ()
else Fmt.pf f ",@,@[<v2>%S: [@ %a@]@ ]" key pp_deps deps

let pp_depexts =
let pp_deps = Fmt.iter ~sep:Fmt.comma String_set.iter Fmt.Dump.string in
fun f deps ->
if String_set.is_empty deps then ()
else Fmt.pf f ",@,@[<v2>\"depexts\": [@ %a@]@ ]" pp_deps deps

let pp_pkg ppf (t : Lock_pkg.t) =
Fmt.pf ppf "\"version\": %S%a%a%a%a%a%a%a"
(OpamPackage.version_to_string t.opam_details.package)
pp_src t (pp_depends "depends") t.depends
(pp_depends "buildDepends")
t.depends_build (pp_depends "testDepends") t.depends_test
(pp_depends "docDepends") t.depends_doc
(pp_depends "devSetupDepends")
t.depends_dev_setup pp_depexts
(String_set.union t.depexts_unknown t.depexts_nix)

(* Lock file printers *)

let pp_version f version = Fmt.pf f "\"version\": %S" version

let pp_repo_uri f repo_url =
match repo_url.OpamUrl.hash with
| Some rev ->
Fmt.pf f "@[<v2>\"repository\": {@ \"url\": %a,@ \"rev\": %S@]@,}"
(Fmt.quote Opam_utils.pp_url)
{ repo_url with OpamUrl.hash = None }
rev
| None ->
Fmt.invalid_arg "Repo URI without fragment: %a" Opam_utils.pp_url repo_url

let pp_packages f deps =
let pp_pkg fmt pkg =
Fmt.pf fmt "@[<v2>%a: {@ %a@]@,}" pp_name_quoted (Lock_pkg.name pkg) pp_pkg
pkg
in
let pp_list = Fmt.iter ~sep:Fmt.comma List.iter pp_pkg in
Fmt.pf f "@[<v2>\"packages\" : {@,%a@]@,}" (Fmt.hvbox pp_list) deps

let pp fmt (t : Lock_file.t) =
Fmt.pf fmt {|{@[<v2>@,%a,@,%a,@,%a@]@,}@.|} pp_version Lib.version pp_repo_uri
t.repo pp_packages t.packages
6 changes: 6 additions & 0 deletions onix/Pp_lock.mli
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
val pp_pkg : Format.formatter -> Lock_pkg.t -> unit

val pp : Format.formatter -> Lock_file.t -> unit
(** Pretty-printer for the nix lock file.
[~ignore_file] is the optional file path to be used to filter root sources. *)
Loading

0 comments on commit 983425a

Please sign in to comment.