Skip to content

Commit

Permalink
* Adds support for private Bazel deps
Browse files Browse the repository at this point in the history
  • Loading branch information
@njlr
njlr committed Sep 19, 2019
1 parent bdae375 commit ed692c0
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 107 deletions.
3 changes: 3 additions & 0 deletions buckaroo/Constants.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ let LockFileName = "buckaroo.lock.toml"
[<Literal>]
let BuckarooMacrosFileName = "buckaroo_macros.bzl"

[<Literal>]
let BuckarooDefsFileName = "defs.bzl"

[<Literal>]
let BuckarooDepsFileName = "BUCKAROO_DEPS"

Expand Down
236 changes: 139 additions & 97 deletions buckaroo/InstallCommand.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ module Buckaroo.InstallCommand

open System
open System.IO
open FSharp.Control
open FSharpx.Control
open Buckaroo
open Buckaroo.BuckConfig
Expand All @@ -19,35 +20,7 @@ let private fetchManifestFromLock (lock : Lock) (sourceExplorer : ISourceExplore
return! sourceExplorer.FetchManifest location
}

let private fetchDependencyTargets (lock : Lock) (sourceExplorer : ISourceExplorer) (manifest : Manifest) = async {
let! targetIdentifiers =
manifest.Dependencies
|> Seq.map (fun d -> async {
let! targets =
match d.Targets with
| Some targets -> async {
return targets |> List.toSeq
}
| None -> async {
let! manifest = fetchManifestFromLock lock sourceExplorer d.Package
return manifest.Targets |> Set.toSeq
}

return
targets
|> Seq.map (fun target ->
{ Package = d.Package; Target = target }
)
})
|> Async.Parallel

return
targetIdentifiers
|> Seq.collect id
|> Seq.toList
}

let private buckarooMacros =
let private buckarooBuckMacros =
[
"def buckaroo_cell(package): ";
" cell = native.read_config('buckaroo', package, '').strip()";
Expand Down Expand Up @@ -91,11 +64,24 @@ let rec computeCellIdentifier (parents : PackageIdentifier list) (package : Pack
| head::tail ->
(computeCellIdentifier tail head) + "." + (computeCellIdentifier [] package)

let rec private combinePaths xs =
match xs with
| [ x ] -> x
| x::xs -> Path.Combine (x, combinePaths xs)
| [] -> ""
let private packageWorkspaceName (parents : PackageIdentifier list) (package : PackageIdentifier) =
let s =
seq {
yield! parents
yield package
}
|> Seq.map PackageIdentifier.show
|> String.concat "_"

"buckaroo_" +
(
s
|> Strings.replaceAll [ "/"; "."; "-" ] "_"
) + "_" +
(
Hashing.sha256 s
|> Strings.truncate 8
)

let rec packageInstallPath (parents : PackageIdentifier list) (package : PackageIdentifier) =
match parents with
Expand All @@ -104,7 +90,7 @@ let rec packageInstallPath (parents : PackageIdentifier list) (package : Package
match package with
| PackageIdentifier.GitHub x -> ("github", x.Owner, x.Project)
| PackageIdentifier.BitBucket x -> ("bitbucket", x.Owner, x.Project)
| PackageIdentifier.GitLab x -> ("gitlab", combinePaths x.Groups, x.Project)
| PackageIdentifier.GitLab x -> ("gitlab", Paths.combineAll x.Groups, x.Project)
| PackageIdentifier.Adhoc x -> ("adhoc", x.Owner, x.Project)
String.Join ("/", [ Constants.PackagesDirectory; prefix; owner; project ])
| head::tail ->
Expand Down Expand Up @@ -345,65 +331,40 @@ let private generateBuckConfig (sourceExplorer : ISourceExplorer) (parents : Pac
return buckarooConfig |> BuckConfig.render
}

let private workspaceName (package : PackageIdentifier) =
"buckaroo_" + (PackageIdentifier.show package)
|> Strings.replace "-" "_"
|> Strings.replace "." "_"
|> Strings.replace "/" "_"

let private dataBzl (packages : Map<PackageIdentifier, LockedPackage>) =
let private generateBazelDefs (targets : Set<TargetIdentifier>) =
seq {
yield "packages = {"
yield "def buckaroo_deps():"

yield " return ["
yield!
packages
|> Map.toSeq
|> Seq.collect (fun (packageIdentifier, lockedPackage) -> seq {
yield " \"" + (PackageIdentifier.show packageIdentifier) + "\": {"
yield " \"workspace\": \"" + (workspaceName packageIdentifier) + "\","
yield " \"path\": \"" + (packageInstallPath [] packageIdentifier) + "\", "
yield " }, "
targets
|> Seq.collect (fun t -> seq {
let parents, package = t.PackagePath
yield " \"@" + (packageWorkspaceName parents package) + (Target.show t.Target) + "\","
})
yield "}"

yield " ]"

yield ""
}
|> String.concat "\n"

let private defsBzl = """
load("//:data.bzl", "packages")
# Utility function to get the name of the Bazel WORKSPACE at which a package is installed.
def buckaroo_workspace(package_name):
package = packages.get(package_name)
if package:
return package.get("workspace")
else:
fail("The package " + package_name + " could not be found. ")
# Utility function to be used in the user's WORKSPACE to wire-up each package.
def buckaroo_setup():
for package in packages.values():
native.local_repository(
name = package.get("workspace"),
path = package.get("path"),
)
"""

let rec private installBuckPackages (context : Tasks.TaskContext) (root : string) (parents : PackageIdentifier list) (packages : Map<PackageIdentifier, LockedPackage>) = async {
// Prepare workspace
do! Files.mkdirp root
do! Files.touch (Path.Combine(root, ".buckconfig"))

if File.Exists (Path.Combine(root, Constants.BuckarooMacrosFileName)) |> not
then
do! Files.writeFile (Path.Combine(root, Constants.BuckarooMacrosFileName)) buckarooMacros
do! Files.writeFile (Path.Combine(root, Constants.BuckarooMacrosFileName)) buckarooBuckMacros

// Install packages
for (package, lockedPackage) in packages |> Map.toSeq do
let installPath =
Path.Combine(root, packageInstallPath [] package)
|> Paths.normalize

let childParents = (parents @ [ package ])
let childParents = parents @ [ package ]

// Install child package sources
do! installPackageSources context installPath lockedPackage.Location lockedPackage.Versions
Expand All @@ -419,37 +380,113 @@ let rec private installBuckPackages (context : Tasks.TaskContext) (root : string
do! Files.writeFile (Path.Combine(installPath, ".buckconfig.d", ".buckconfig.buckaroo")) buckarooConfig
}

let rec private computeNestedPackages (parents : PackageIdentifier list) packages =
packages
|> Map.toSeq
|> Seq.collect (fun (k, v) -> seq {
yield (parents, k)
yield! computeNestedPackages (parents @ [ k ]) v.PrivatePackages
})

let private generateTopLevelBazelDefs (sourceExplorer : ISourceExplorer) (lock : Lock) = async {
return
seq {
yield "def buckaroo_setup():"

// We need to create a "local_repository" for every package and its private packages
yield!
lock.Packages
|> computeNestedPackages []
|> Seq.collect (fun (parents, package) -> seq {
yield " native.local_repository("
yield " name = \"" + (packageWorkspaceName parents package) + "\","
yield " path = \"" + (packageInstallPath parents package) + "\""
yield " )"
yield ""
})

yield " return None"
yield ""
yield generateBazelDefs lock.Dependencies
}
|> String.concat "\n"
}

let rec private installBazelPackages (context : Tasks.TaskContext) (root : string) (parents : PackageIdentifier list) (packages : Map<PackageIdentifier, LockedPackage>) = async {
// Prepare workspace
do! Files.mkdirp root
do! Files.touch (Path.Combine (root, "WORKSPACE"))
do! Files.touch (Paths.combine root "WORKSPACE")

// Install packages
for (package, lockedPackage) in packages |> Map.toSeq do
let installPath =
Path.Combine(root, packageInstallPath [] package)
Paths.combine root (packageInstallPath [] package)
|> Paths.normalize

let childParents = (parents @ [ package ])
let childParents = parents @ [ package ]

// Install child package sources
do! installPackageSources context installPath lockedPackage.Location lockedPackage.Versions

// Install child's child (recurse)
// Install child's children (recurse)
do! installBazelPackages context installPath childParents lockedPackage.PrivatePackages

// Write macros
do! Files.writeFile (Path.Combine ("buckaroo", "data.bzl")) (dataBzl packages)
do! Files.writeFile (Path.Combine ("buckaroo", "defs.bzl")) defsBzl
}
// Write buckaroo/defs.bzl
do! Files.mkdirp (Paths.combineAll [ installPath; Constants.PackagesDirectory ])

do! Files.touch (Paths.combineAll [ installPath; Constants.PackagesDirectory; "BUILD.bazel" ])

let! manifest = context.SourceExplorer.FetchManifest (lockedPackage.Location, lockedPackage.Versions)

let! targets =
asyncSeq {
let xs =
manifest.Dependencies
|> Seq.map (fun d -> (parents, d))
|> Seq.append (
manifest.PrivateDependencies
|> Seq.map (fun d -> (parents @ [ package ], d))
)

for parents, dependency in xs do
match dependency.Targets with
| Some targets ->
yield!
targets
|> Seq.map (fun target ->
{
PackagePath = parents, dependency.Package
Target = target
}
)
|> AsyncSeq.ofSeq
| None ->
// If the manifest does not specify any targets then
// we need to take the defaults listed in the package manifest.
let lockedPackage =
packages
|> Map.find dependency.Package // TODO: Better error if this fails. Indicates an invalid lock-file?

let! manifest =
context.SourceExplorer.FetchManifest (lockedPackage.Location, lockedPackage.Versions)

yield!
manifest.Targets
|> Seq.map (fun target ->
{
PackagePath = parents, dependency.Package
Target = target
}
)
|> AsyncSeq.ofSeq
}
|> AsyncSeq.toListAsync
|> Async.map Set.ofSeq

let buckarooDefs =
generateBazelDefs
targets

let rec private computeNestedPackages (parents : PackageIdentifier list) packages =
packages
|> Map.toSeq
|> Seq.collect (fun (k, v) -> seq {
yield (parents, k)
yield! computeNestedPackages (parents @ [ k ]) v.PrivatePackages
})
do! Files.writeFile (Path.Combine(installPath, Constants.PackagesDirectory, Constants.BuckarooDefsFileName)) buckarooDefs
}

let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock : Lock) = async {
let nestedPackages =
Expand All @@ -476,7 +513,7 @@ let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock :
"dependencies",
(
lock.Dependencies
|> Seq.map (fun d -> (computeCellIdentifier [] d.Package) + (Target.show d.Target))
|> Seq.map (fun d -> (computeCellIdentifier (fst d.PackagePath) (snd d.PackagePath)) + (Target.show d.Target))
|> String.concat " "
|> INIString
)
Expand All @@ -493,6 +530,15 @@ let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock :
do! Files.writeFile (Path.Combine (root, ".buckconfig.d", ".buckconfig.buckaroo")) (BuckConfig.render config)
}

let writeTopLevelBazelFiles context lock = async {
do! Files.mkdirp (Paths.combine "." Constants.PackagesDirectory)
do! Files.touch (Paths.combineAll [ "."; Constants.PackagesDirectory; "BUILD.bazel" ])

let! bazelDefs = generateTopLevelBazelDefs context.SourceExplorer lock

do! Files.writeFile (Paths.combineAll [ "."; Constants.PackagesDirectory; Constants.BuckarooDefsFileName ]) bazelDefs
}

let task (context : Tasks.TaskContext) = async {
let logger = createLogger context.Console None

Expand All @@ -506,15 +552,11 @@ let task (context : Tasks.TaskContext) = async {

match context.BuildSystem with
| Buck ->
do! installBuckPackages context "." [] lock.Packages
do! writeTopLevelBuckFiles context "." lock
do! installBuckPackages context "." [] lock.Packages
| Bazel ->
do! Files.mkdirp "buckaroo"
do! Files.touch (Paths.combine "buckaroo" "WORKSPACE")
do! Files.touch (Paths.combine "buckaroo" "BUILD")
do! writeTopLevelBazelFiles context lock
do! installBazelPackages context "." [] lock.Packages
// failwith "Not implemented"
// TODO

logger.Success "The packages folder is now up-to-date. "

Expand Down
10 changes: 7 additions & 3 deletions buckaroo/Lock.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ module Lock =
|> Map.tryFind p
|> Option.map (fun (rv, _) ->
rv.Manifest.Targets
|> Seq.map (fun t -> { Package = p; Target = t })
|> Seq.map (fun t -> { PackagePath = [], p; Target = t })
)
|> Option.defaultValue Seq.empty
)
Expand Down Expand Up @@ -187,7 +187,7 @@ module Lock =
lock.Dependencies
|> Seq.map(fun x ->
"[[dependency]]\n" +
"package = \"" + (PackageIdentifier.show x.Package) + "\"\n" +
"package = \"" + (PackageIdentifier.show (snd x.PackagePath)) + "\"\n" +
"target = \"" + (Target.show x.Target) + "\"\n\n"
)
|> String.concat ""
Expand Down Expand Up @@ -415,7 +415,11 @@ module Lock =
|> Result.bind (Toml.asString >> Result.mapError Toml.TomlError.show)
|> Result.bind Target.parse

return { Package = package; Target = target }
return
{
PackagePath = [], package
Target = target
}
}

let parse (content : string) : Result<Lock, string> = result {
Expand Down
2 changes: 1 addition & 1 deletion buckaroo/PackageLocation.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,4 @@ module PackageLocation =
| PackageLocation.Git g -> Set [Version.Git (GitVersion.Revision g.Revision)]
| PackageLocation.GitLab g -> Set [Version.Git (GitVersion.Revision g.Revision)]
| PackageLocation.BitBucket g -> Set [Version.Git (GitVersion.Revision g.Revision)]
| _ -> Set.empty
| _ -> Set.empty
Loading

0 comments on commit ed692c0

Please sign in to comment.