mill update

Makefile

@@ -53,7 +53,7 @@ pylint-hailtop:
 
 .PHONY: check-hail
 check-hail: check-hail-fast pylint-hailtop
-	cd hail && sh millw __.checkFormat + __.fix --check
+	cd hail && sh mill __.checkFormat + __.fix --check
 
 .PHONY: check-services
 check-services: $(CHECK_SERVICES_MODULES)

build.yaml

@@ -724,7 +724,7 @@ steps:
       cd /io/repo/hail
 
       export MILLOPTS='--no-server' HAIL_RELEASE_MODE=1
-      time retry sh millw --no-server --version
+      time retry sh mill --no-server --version
 
       # We've encountered the following sporadic error in CI between `mill`
       # finishing assembling the jar and `Make` copying the jar:
@@ -770,7 +770,7 @@ steps:
       cd /io/repo/hail
 
       export MILLOPTS='--no-server'
-      time retry sh millw --no-server --version
+      time retry sh mill --no-server --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
@@ -803,7 +803,7 @@ steps:
       cd /io/repo/hail
 
       export MILLOPTS='--no-server'
-      time retry sh millw --no-server --version
+      time retry sh mill --no-server --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowTestJar
@@ -863,7 +863,7 @@ steps:
       cd /io/repo/hail
 
       export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_RELEASE_MODE=1 MILLOPTS='--no-server'
-      time retry sh millw --no-server --version
+      time retry sh mill --no-server --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar

hail/.mill-version

@@ -1 +1 @@
-0.11.7
+0.12.4

hail/Makefile

@@ -49,7 +49,7 @@ CLOUD_HAIL_DOCTEST_DATA_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/doctest/data/
 
 # mill integration
 
-MILL := bash millw
+MILL := bash mill
 MILLOPTS ?=
 
 .PHONY: mill-clean

hail/build.sc → hail/build.mill

@@ -1,5 +1,7 @@
-import $ivy.`com.goyeau::mill-scalafix::0.3.2`
-import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
+package build
+
+import $ivy.`com.goyeau::mill-scalafix::0.4.2`
+import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.1`
 import com.goyeau.mill.scalafix.ScalafixModule
 import de.tobiasroeser.mill.vcs.version.VcsVersion
 import mill._
@@ -15,56 +17,6 @@ object Settings {
   val hailPatchVersion = "133"
 }
 
-/** Update the millw script. */
-def millw(): Command[PathRef] = T.command {
-  val target =
-    mill.util.Util.download("https://raw.githubusercontent.com/lefou/millw/main/millw")
-  val millw = T.workspace / "millw"
-  os.copy.over(target.path, millw)
-  os.perms.set(millw, os.perms(millw) + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)
-  target
-}
-
-def scalaVersion: T[String] = T.input {
-  val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.15")
-  if (!v.startsWith("2.12"))
-    Result.Failure("Hail currently supports only Scala 2.12")
-  else
-    v
-}
-
-def javaVersion: T[String] = T.input {
-  System.getProperty("java.version")
-}
-
-def sparkVersion: T[String] = T.input {
-  Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
-}
-
-def debugMode: T[Boolean] = T.input {
-  val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
-  T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
-  isDebug
-}
-
-def debugOrRelease: Task[String] = T.task {
-  if (debugMode()) "debug" else "release"
-}
-
-def buildInfo: T[PathRef] = T {
-  val revision = VcsVersion.vcsState().currentRevision
-  os.write(
-    T.dest / "build-info.properties",
-    s"""[Build Metadata]
-       |revision=$revision
-       |sparkVersion=${sparkVersion()}
-       |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
-       |hailBuildConfiguration=${debugOrRelease()}
-       |""".stripMargin,
-  )
-  PathRef(T.dest)
-}
-
 object Deps {
   object HTTPComponents {
     val core = ivy"org.apache.httpcomponents:httpcore:4.4.14"
@@ -94,8 +46,8 @@ object Deps {
   }
 
   object Spark {
-    def core: Task[Dep] = T.task(ivy"org.apache.spark::spark-core:${sparkVersion()}")
-    def mllib: Task[Dep] = T.task(ivy"org.apache.spark::spark-mllib:${sparkVersion()}")
+    def core: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-core:${build.env.sparkVersion()}")
+    def mllib: Task[Dep] = Task.Anon(ivy"org.apache.spark::spark-mllib:${build.env.sparkVersion()}")
   }
 
   val samtools = ivy"com.github.samtools:htsjdk:3.0.5"
@@ -121,13 +73,13 @@ object Deps {
 }
 
 trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule { outer =>
-  override def scalaVersion: T[String] = build.scalaVersion()
+  override def scalaVersion: T[String] = build.env.scalaVersion()
 
   override def javacOptions: T[Seq[String]] = Seq(
     "-Xlint:all",
     "-Werror",
-    if (debugMode()) "-g" else "-O",
-  ) ++ (if (!javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())
+    if (build.env.debugMode()) "-g" else "-O",
+  ) ++ (if (!build.env.javaVersion().startsWith("1.8")) Seq("-Xlint:-processing") else Seq())
 
   override def scalacOptions: T[Seq[String]] = T {
     Seq(
@@ -139,8 +91,9 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
       "-Yno-adapted-args", // will be removed in 2.13
       "-Xlint",
       "-Ywarn-unused:_,-explicits,-implicits",
+      "-Wconf:msg=legacy-binding:s",
     ) ++ (
-      if (debugMode()) Seq()
+      if (build.env.debugMode()) Seq()
       else Seq(
         "-Xfatal-warnings",
         "-opt:l:method",
@@ -153,7 +106,7 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
   override def bspCompileClasspath: T[Agg[UnresolvedPath]] =
     super.bspCompileClasspath() ++ resources().map(p => UnresolvedPath.ResolvedPath(p.path))
 
-  trait HailTests extends SbtModuleTests with TestNg with ScalafmtModule with ScalafixModule {
+  trait HailTests extends SbtTests with TestNg with ScalafmtModule with ScalafixModule {
     override def forkArgs: T[Seq[String]] = Seq("-Xss4m", "-Xmx4096M")
 
     override def ivyDeps: T[Agg[Dep]] =
@@ -171,7 +124,50 @@ trait HailScalaModule extends SbtModule with ScalafmtModule with ScalafixModule
   }
 }
 
-object main extends RootModule with HailScalaModule { outer =>
+object `package` extends RootModule with HailScalaModule { outer =>
+
+  object env extends Module {
+    def scalaVersion: T[String] = Task.Input {
+      val v = T.ctx().env.getOrElse("SCALA_VERSION", "2.12.20")
+      if (!v.startsWith("2.12"))
+        Result.Failure("Hail currently supports only Scala 2.12")
+      else
+        v
+    }
+
+    def javaVersion: T[String] = Task.Input {
+      System.getProperty("java.version")
+    }
+
+    def sparkVersion: T[String] = Task.Input {
+      Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
+    }
+
+    def debugMode: T[Boolean] = Task.Input {
+      val isDebug = !T.ctx().env.contains("HAIL_RELEASE_MODE")
+      T.log.info(s"Building in ${if (isDebug) "debug" else "release"} mode")
+      isDebug
+    }
+
+    def debugOrRelease: Task[String] = Task.Anon {
+      if (debugMode()) "debug" else "release"
+    }
+  }
+
+  def buildInfo: T[PathRef] = T {
+    val revision = VcsVersion.vcsState().currentRevision
+    os.write(
+      T.dest / "build-info.properties",
+      s"""[Build Metadata]
+         |revision=$revision
+         |sparkVersion=${env.sparkVersion()}
+         |hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
+         |hailBuildConfiguration=${env.debugOrRelease()}
+         |""".stripMargin,
+    )
+    PathRef(T.dest)
+  }
+
   override def moduleDeps: Seq[JavaModule] = Seq(memory)
 
   override def resources: T[Seq[PathRef]] = Seq(
@@ -256,15 +252,15 @@ object main extends RootModule with HailScalaModule { outer =>
 
     override def javacOptions: T[Seq[String]] =
       outer.javacOptions() ++ (
-        if (javaVersion().startsWith("1.8")) Seq(
+        if (env.javaVersion().startsWith("1.8")) Seq(
           "-XDenableSunApiLintControl",
           "-Xlint:-sunapi",
         )
         else Seq()
       )
 
-    override def sources: T[Seq[PathRef]] = T.sources {
-      Seq(PathRef(this.millSourcePath / os.up / "src" / debugOrRelease() / "java"))
+    override def sources: T[Seq[PathRef]] = Task.Sources {
+      Seq(PathRef(this.millSourcePath / os.up / "src" / env.debugOrRelease() / "java"))
     }
   }
 

hail/mill

@@ -0,0 +1,67 @@
+#!/usr/bin/env sh
+
+# This is a wrapper script, that automatically download mill from GitHub release pages
+# You can give the required mill version with MILL_VERSION env variable
+# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION
+
+set -e
+
+if [ -z "${DEFAULT_MILL_VERSION}" ] ; then
+  DEFAULT_MILL_VERSION=0.12.3
+fi
+
+if [ -z "$MILL_VERSION" ] ; then
+  if [ -f ".mill-version" ] ; then
+    MILL_VERSION="$(tr '\r' '\n' < .mill-version | head -n 1 2> /dev/null)"
+  elif [ -f ".config/mill-version" ] ; then
+    MILL_VERSION="$(tr '\r' '\n' < .config/mill-version | head -n 1 2> /dev/null)"
+  elif [ -f "mill" ] && [ "$0" != "mill" ] ; then
+    MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2)
+  else
+    MILL_VERSION=$DEFAULT_MILL_VERSION
+  fi
+fi
+
+if [ "x${XDG_CACHE_HOME}" != "x" ] ; then
+  MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download"
+else
+  MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download"
+fi
+MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}"
+
+version_remainder="$MILL_VERSION"
+MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"
+MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}"
+
+if [ ! -s "$MILL_EXEC_PATH" ] ; then
+  mkdir -p "$MILL_DOWNLOAD_PATH"
+  if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then
+    ASSEMBLY="-assembly"
+  fi
+  DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download
+  MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/')
+  MILL_DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist/$MILL_VERSION/mill-dist-$MILL_VERSION.jar"
+  curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL"
+  chmod +x "$DOWNLOAD_FILE"
+  mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH"
+  unset DOWNLOAD_FILE
+  unset MILL_DOWNLOAD_URL
+fi
+
+if [ -z "$MILL_MAIN_CLI" ] ; then
+  MILL_MAIN_CLI="${0}"
+fi
+
+MILL_FIRST_ARG=""
+
+ # first arg is a long flag for "--interactive" or starts with "-i"
+if [ "$1" = "--bsp" ] || [ "${1#"-i"}" != "$1" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then
+  # Need to preserve the first position of those listed options
+  MILL_FIRST_ARG=$1
+  shift
+fi
+
+unset MILL_DOWNLOAD_PATH
+unset MILL_VERSION
+
+exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@"