Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mill update #14769

Merged
merged 1 commit into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ pylint-hailtop:

.PHONY: check-hail
check-hail: check-hail-fast pylint-hailtop
cd hail && sh millw __.checkFormat + __.fix --check
cd hail && sh mill __.checkFormat + __.fix --check

.PHONY: check-services
check-services: $(CHECK_SERVICES_MODULES)
Expand Down
8 changes: 4 additions & 4 deletions build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ steps:
cd /io/repo/hail

export MILLOPTS='--no-server' HAIL_RELEASE_MODE=1
time retry sh millw --no-server --version
time retry sh mill --no-server --version

# We've encountered the following sporadic error in CI between `mill`
# finishing assembling the jar and `Make` copying the jar:
Expand Down Expand Up @@ -770,7 +770,7 @@ steps:
cd /io/repo/hail

export MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version

# See `build_hail_jar_and_wheel`
time retry make shadowJar
Expand Down Expand Up @@ -803,7 +803,7 @@ steps:
cd /io/repo/hail

export MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version

# See `build_hail_jar_and_wheel`
time retry make shadowTestJar
Expand Down Expand Up @@ -863,7 +863,7 @@ steps:
cd /io/repo/hail

export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_RELEASE_MODE=1 MILLOPTS='--no-server'
time retry sh millw --no-server --version
time retry sh mill --no-server --version

# See `build_hail_jar_and_wheel`
time retry make shadowJar
Expand Down
2 changes: 1 addition & 1 deletion hail/.mill-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.11.7
0.12.4
2 changes: 1 addition & 1 deletion hail/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ CLOUD_HAIL_DOCTEST_DATA_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/doctest/data/

# mill integration

MILL := bash millw
MILL := bash mill
MILLOPTS ?=

.PHONY: mill-clean
Expand Down
122 changes: 59 additions & 63 deletions hail/build.sc → hail/build.mill
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import $ivy.`com.goyeau::mill-scalafix::0.3.2`
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
package build

import $ivy.`com.goyeau::mill-scalafix::0.4.2`
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.1`
import com.goyeau.mill.scalafix.ScalafixModule
import de.tobiasroeser.mill.vcs.version.VcsVersion
import mill._
Expand All @@ -15,56 +17,6 @@ object Settings {
val hailPatchVersion = "133"
}

/** Update the millw script. */
def millw(): Command[PathRef] = T.command {
val target =
mill.util.Util.download("https://raw.githubusercontent.com/lefou/millw/main/millw")
val millw = T.workspace / "millw"
os.copy.over(target.path, millw)
os.perms.set(millw, os.perms(millw) + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)
target
}

def scalaVersion: T[String] = T.input {
val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.15")
if (!v.startsWith("2.12"))
Result.Failure("Hail currently supports only Scala 2.12")
else
v
}

def javaVersion: T[String] = T.input {
System.getProperty("java.version")
}

def sparkVersion: T[String] = T.input {
Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
}

def debugMode: T[Boolean] = T.input {
val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
isDebug
}

def debugOrRelease: Task[String] = T.task {
if (debugMode()) "debug" else "release"
}

def buildInfo: T[PathRef] = T {
val revision = VcsVersion.vcsState().currentRevision
os.write(
T.dest / "build-info.properties",
s"""[Build Metadata]
|revision=$revision
|sparkVersion=${sparkVersion()}
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
|hailBuildConfiguration=${debugOrRelease()}
|""".stripMargin,
)
PathRef(T.dest)
}

object Deps {
object HTTPComponents {
val core = ivy"org.apache.httpcomponents:httpcore:4.4.14"
Expand Down Expand Up @@ -94,8 +46,8 @@ object Deps {
}

object Spark {
def core: Task[Dep] = T.task(ivy"org.apache.spark::spark-core:${sparkVersion()}")
def mllib: Task[Dep] = T.task(ivy"org.apache.spark::spark-mllib:${sparkVersion()}")
def core: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-core:${build.env.sparkVersion()}")
def mllib: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-mllib:${build.env.sparkVersion()}")
}

val samtools = ivy"com.github.samtools:htsjdk:3.0.5"
Expand All @@ -121,13 +73,13 @@ object Deps {
}

trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule { outer =>
override def scalaVersion: T[String] = build.scalaVersion()
override def scalaVersion: T[String] = build.env.scalaVersion()

override def javacOptions: T[Seq[String]] = Seq(
"-Xlint:all",
"-Werror",
if (debugMode()) "-g" else "-O",
) ++ (if (!javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())
if (build.env.debugMode()) "-g" else "-O",
) ++ (if (!build.env.javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())

override def scalacOptions: T[Seq[String]] = T {
Seq(
Expand All @@ -139,8 +91,9 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
"-Yno-adapted-args", // will be removed in 2.13
"-Xlint",
"-Ywarn-unused:_,-explicits,-implicits",
"-Wconf:msg=legacy-binding:s",
) ++ (
if (debugMode()) Seq()
if (build.env.debugMode()) Seq()
else Seq(
"-Xfatal-warnings",
"-opt:l:method",
Expand All @@ -153,7 +106,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
override def bspCompileClasspath: T[Agg[UnresolvedPath]] =
super.bspCompileClasspath() ++ resources().map(p => UnresolvedPath.ResolvedPath(p.path))

trait HailTests extends SbtModuleTests with TestNg with ScalafmtModule with ScalafixModule {
trait HailTests extends SbtTests with TestNg with ScalafmtModule with ScalafixModule {
override def forkArgs: T[Seq[String]] = Seq("-Xss4m", "-Xmx4096M")

override def ivyDeps: T[Agg[Dep]] =
Expand All @@ -171,7 +124,50 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
}
}

object main extends RootModule with HailScalaModule { outer =>
object `package` extends RootModule with HailScalaModule { outer =>

object env extends Module {
def scalaVersion: T[String] = Task.Input {
val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.20")
if (!v.startsWith("2.12"))
Result.Failure("Hail currently supports only Scala 2.12")
else
v
}

def javaVersion: T[String] = Task.Input {
System.getProperty("java.version")
}

def sparkVersion: T[String] = Task.Input {
Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
}

def debugMode: T[Boolean] = Task.Input {
val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
isDebug
}

def debugOrRelease: Task[String] = Task.Anon {
if (debugMode()) "debug" else "release"
}
}

def buildInfo: T[PathRef] = T {
val revision = VcsVersion.vcsState().currentRevision
os.write(
T.dest / "build-info.properties",
s"""[Build Metadata]
|revision=$revision
|sparkVersion=${env.sparkVersion()}
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
|hailBuildConfiguration=${env.debugOrRelease()}
|""".stripMargin,
)
PathRef(T.dest)
}

override def moduleDeps: Seq[JavaModule] = Seq(memory)

override def resources: T[Seq[PathRef]] = Seq(
Expand Down Expand Up @@ -256,15 +252,15 @@ object main extends RootModule with HailScalaModule { outer =>

override def javacOptions: T[Seq[String]] =
outer.javacOptions() ++ (
if (javaVersion().startsWith("1.8")) Seq(
if (env.javaVersion().startsWith("1.8")) Seq(
"-XDenableSunApiLintControl",
"-Xlint:-sunapi",
)
else Seq()
)

override def sources: T[Seq[PathRef]] = T.sources {
Seq(PathRef(this.millSourcePath / os.up / "src" / debugOrRelease() / "java"))
override def sources: T[Seq[PathRef]] = Task.Sources {
Seq(PathRef(this.millSourcePath / os.up / "src" / env.debugOrRelease() / "java"))
}
}

Expand Down
67 changes: 67 additions & 0 deletions hail/mill
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env sh

# This is a wrapper script, that automatically download mill from GitHub release pages
# You can give the required mill version with MILL_VERSION env variable
# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION

set -e

if [ -z "${DEFAULT_MILL_VERSION}" ] ; then
DEFAULT_MILL_VERSION=0.12.3
fi

if [ -z "$MILL_VERSION" ] ; then
if [ -f ".mill-version" ] ; then
MILL_VERSION="$(tr '\r' '\n' < .mill-version | head -n 1 2> /dev/null)"
elif [ -f ".config/mill-version" ] ; then
MILL_VERSION="$(tr '\r' '\n' < .config/mill-version | head -n 1 2> /dev/null)"
elif [ -f "mill" ] && [ "$0" != "mill" ] ; then
MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2)
else
MILL_VERSION=$DEFAULT_MILL_VERSION
fi
fi

if [ "x${XDG_CACHE_HOME}" != "x" ] ; then
MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download"
else
MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download"
fi
MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}"

version_remainder="$MILL_VERSION"
MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"
MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"

if [ ! -s "$MILL_EXEC_PATH" ] ; then
mkdir -p "$MILL_DOWNLOAD_PATH"
if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then
ASSEMBLY="-assembly"
fi
DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download
MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/')
MILL_DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist/$MILL_VERSION/mill-dist-$MILL_VERSION.jar"
curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL"
chmod +x "$DOWNLOAD_FILE"
mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH"
unset DOWNLOAD_FILE
unset MILL_DOWNLOAD_URL
fi

if [ -z "$MILL_MAIN_CLI" ] ; then
MILL_MAIN_CLI="${0}"
fi

MILL_FIRST_ARG=""

# first arg is a long flag for "--interactive" or starts with "-i"
if [ "$1" = "--bsp" ] || [ "${1#"-i"}" != "$1" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then
# Need to preserve the first position of those listed options
MILL_FIRST_ARG=$1
shift
fi

unset MILL_DOWNLOAD_PATH
unset MILL_VERSION

exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@"
Loading