Skip to content

Commit

Permalink
Separate Docker Layers for Dependencies and App jars. (#1310)
Browse files Browse the repository at this point in the history
* Docker Layers separate for Dependencies and App jars.

* Use indexed layers to allow flexibility

* avoid mapping `dockerPackageMappings`

* Calculate `dockerLayerMappings` once

It is reused in Docker/stage and and Docker/dockerCommands.
This intermediate step helps with cyclic dependency.

* Dirty version that supports PR code review remarks

* Fix chmod for layers

* Revert changes around Docker / mappings

* Revert changes dockerLayerMappings

* Documenting `dockerLayerGrouping`

* Testing layer groups

* fix tests

* fix more tests

* Version that might pass the tests

* Fix import for sbt 0.1.x

* scalafmt

* Experiment with Option[Int] for layer mapping

* Docker tests of opt-out from layering

* Test of dockerPackageMappings

* Documentation
  • Loading branch information
ppiotrow authored Mar 16, 2020
1 parent 980273b commit 411bb5a
Show file tree
Hide file tree
Showing 16 changed files with 183 additions and 48 deletions.
108 changes: 78 additions & 30 deletions src/main/scala/com/typesafe/sbt/packager/docker/DockerPlugin.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import java.util.UUID
import java.util.concurrent.atomic.AtomicBoolean

import sbt._
import sbt.Keys.{clean, mappings, name, publish, publishLocal, sourceDirectory, streams, target, version}
import sbt.Keys.{clean, mappings, name, organization, publish, publishLocal, sourceDirectory, streams, target, version}
import com.typesafe.sbt.packager.Keys._
import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport.{daemonUser, defaultLinuxInstallLocation}
import com.typesafe.sbt.packager.universal.UniversalPlugin
Expand Down Expand Up @@ -58,7 +58,7 @@ object DockerPlugin extends AutoPlugin {
import autoImport._

/**
* The separator used by makeCopy should be always forced to UNIX separator.
* The separator used by makeCopyLayerIntermediate should be always forced to UNIX separator.
* The separator doesn't depend on the OS where Dockerfile is being built.
*/
val UnixSeparatorChar = '/'
Expand Down Expand Up @@ -98,6 +98,20 @@ object DockerPlugin extends AutoPlugin {
),
dockerUpdateLatest := false,
dockerAutoremoveMultiStageIntermediateImages := true,
dockerLayerGrouping := {
val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
(path: String) =>
{
val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
Some(2)
else if (pathInWorkdir.startsWith("/lib/"))
Some(1)
else if (pathInWorkdir.startsWith("/bin/"))
Some(1)
else None
}
},
dockerAliases := {
val alias = dockerAlias.value
if (dockerUpdateLatest.value) {
Expand All @@ -123,13 +137,14 @@ object DockerPlugin extends AutoPlugin {
dockerBuildCommand := dockerExecCommand.value ++ Seq("build") ++ dockerBuildOptions.value ++ Seq("."),
dockerAdditionalPermissions := {
val basePath = (defaultLinuxInstallLocation in Docker).value
(mappings in Docker).value
(dockerLayerMappings in Docker).value
.collect {
// by default we assume everything in the bin/ folder should be executable that is not a .bat file
case (_, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
case LayeredMapping(_, _, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
DockerChmodType.UserGroupPlusExecute -> path
// sh files should also be marked as executable
case (_, path) if path.endsWith(".sh") => DockerChmodType.UserGroupPlusExecute -> path
case LayeredMapping(_, _, path) if path.endsWith(".sh") =>
DockerChmodType.UserGroupPlusExecute -> path
}
},
dockerCommands := {
Expand All @@ -142,21 +157,28 @@ object DockerPlugin extends AutoPlugin {
val base = dockerBaseImage.value
val addPerms = dockerAdditionalPermissions.value
val multiStageId = UUID.randomUUID().toString

val generalCommands = makeFrom(base) +: makeMaintainer((maintainer in Docker).value).toSeq
val stage0name = "stage0"
val layerIdsAscending = (dockerLayerMappings in Docker).value.map(_.layerId).distinct.sorted
val stage0: Seq[CmdLike] = strategy match {
case DockerPermissionStrategy.MultiStage =>
Seq(
makeFromAs(base, stage0name),
makeLabel("snp-multi-stage" -> "intermediate"),
makeLabel("snp-multi-stage-id" -> multiStageId),
makeWorkdir(dockerBaseDirectory),
makeCopy(dockerBaseDirectory),
makeUser("root"),
makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))
makeWorkdir(dockerBaseDirectory)
) ++
(addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) }) ++
layerIdsAscending.map(l => makeCopyLayerIntermediate(l, dockerBaseDirectory)) ++
Seq(makeUser("root")) ++ layerIdsAscending.map(
l => makeChmodRecursive(dockerChmodType.value, Seq(pathInLayer(dockerBaseDirectory, l)))
) ++ {
val layerToPath = (dockerLayerGrouping in Docker).value
addPerms map {
case (tpe, v) =>
val layerId = layerToPath(v)
makeChmod(tpe, Seq(pathInLayer(v, layerId)))
}
} ++
Seq(DockerStageBreak)
case _ => Seq()
}
Expand All @@ -166,18 +188,22 @@ object DockerPlugin extends AutoPlugin {
case Some(_) => Seq(makeUser("root"), makeUserAdd(user, group, uidOpt, gidOpt))
case _ => Seq()
}) ++
Seq(makeWorkdir(dockerBaseDirectory)) ++
Seq(makeWorkdir(dockerBaseDirectory)) ++ {
(strategy match {
case DockerPermissionStrategy.MultiStage =>
Seq(makeCopyFrom(dockerBaseDirectory, stage0name, user, group))
layerIdsAscending.map { layerId =>
makeCopyFrom(pathInLayer(dockerBaseDirectory, layerId), dockerBaseDirectory, stage0name, user, group)
}
case DockerPermissionStrategy.Run =>
Seq(makeCopy(dockerBaseDirectory), makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))) ++
layerIdsAscending.map(layerId => makeCopyLayerDirect(layerId, dockerBaseDirectory)) ++
Seq(makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))) ++
(addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) })
case DockerPermissionStrategy.CopyChown =>
Seq(makeCopyChown(dockerBaseDirectory, user, group))
layerIdsAscending.map(layerId => makeCopyChown(layerId, dockerBaseDirectory, user, group))
case DockerPermissionStrategy.None =>
Seq(makeCopy(dockerBaseDirectory))
}) ++
layerIdsAscending.map(layerId => makeCopyLayerDirect(layerId, dockerBaseDirectory))
})
} ++
dockerLabels.value.map(makeLabel) ++
dockerEnvVars.value.map(makeEnvVar) ++
makeExposePorts(dockerExposedPorts.value, dockerExposedUdpPorts.value) ++
Expand Down Expand Up @@ -231,9 +257,19 @@ object DockerPlugin extends AutoPlugin {
}
},
sourceDirectory := sourceDirectory.value / "docker",
stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, mappings.value),
stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, dockerLayerMappings.value.map {
case LayeredMapping(layerIdx, file, path) => (file, pathInLayer(path, layerIdx))
}),
stage := (stage dependsOn dockerGenerateConfig).value,
stagingDirectory := (target in Docker).value / "stage",
dockerLayerMappings := {
val dockerGroups = dockerLayerGrouping.value
val dockerFinalFiles = (mappings in Docker).value
for {
(file, path) <- dockerFinalFiles
layerIdx = dockerGroups(path)
} yield LayeredMapping(layerIdx, file, path)
},
target := target.value / "docker",
// pick a user name that's unlikely to exist in base images
daemonUser := "demiourgos728",
Expand Down Expand Up @@ -316,46 +352,57 @@ object DockerPlugin extends AutoPlugin {
* @param dockerBaseDirectory the installation directory
* @return COPY command copying all files inside the installation directory
*/
private final def makeCopy(dockerBaseDirectory: String): CmdLike = {
private final def makeCopyLayerDirect(layerId: Option[Int], dockerBaseDirectory: String): CmdLike = {

/**
* This is the file path of the file in the Docker image, and does not depend on the OS where the image
* is being built. This means that it needs to be the Unix file separator even when the image is built
* on e.g. Windows systems.
*/
val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
Cmd("COPY", s"$files /$files")
val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
Cmd("COPY", s"$path /$files")
}

private final def makeCopyLayerIntermediate(layerId: Option[Int], dockerBaseDirectory: String): CmdLike = {
val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
Cmd("COPY", s"$path /$path")
}

/**
* @param dockerBaseDirectory the installation directory
* @param from files are copied from the given build stage
* @param src the installation directory
* @param stage files are copied from the given build stage
* @param daemonUser
* @param daemonGroup
* @return COPY command copying all files inside the directory from another build stage.
*/
private final def makeCopyFrom(dockerBaseDirectory: String,
from: String,
private final def makeCopyFrom(src: String,
dest: String,
stage: String,
daemonUser: String,
daemonGroup: String): CmdLike =
Cmd("COPY", s"--from=$from --chown=$daemonUser:$daemonGroup $dockerBaseDirectory $dockerBaseDirectory")
Cmd("COPY", s"--from=$stage --chown=$daemonUser:$daemonGroup $src $dest")

/**
* @param dockerBaseDirectory the installation directory
* @param from files are copied from the given build stage
* @param layerId the intermediate layer
* @param daemonUser
* @param daemonGroup
* @return COPY command copying all files inside the directory from another build stage.
*/
private final def makeCopyChown(dockerBaseDirectory: String, daemonUser: String, daemonGroup: String): CmdLike = {
private final def makeCopyChown(layerId: Option[Int],
dockerBaseDirectory: String,
daemonUser: String,
daemonGroup: String): CmdLike = {

/**
* This is the file path of the file in the Docker image, and does not depend on the OS where the image
* is being built. This means that it needs to be the Unix file separator even when the image is built
* on e.g. Windows systems.
*/
val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
Cmd("COPY", s"--chown=$daemonUser:$daemonGroup $files /$files")
val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
Cmd("COPY", s"--chown=$daemonUser:$daemonGroup $path /$files")
}

/**
Expand Down Expand Up @@ -501,6 +548,8 @@ object DockerPlugin extends AutoPlugin {
inConfig(Docker)(Seq(mappings := renameDests((mappings in Universal).value, defaultLinuxInstallLocation.value)))
}

private final def pathInLayer(path: String, layer: Option[Int]) = layer.map(i => s"/$i$path").getOrElse(path)

private[packager] def publishLocalLogger(log: Logger) =
new sys.process.ProcessLogger {
override def err(err: => String): Unit =
Expand Down Expand Up @@ -716,5 +765,4 @@ object DockerPlugin extends AutoPlugin {
case _ => List.empty
}
}

}
6 changes: 6 additions & 0 deletions src/main/scala/com/typesafe/sbt/packager/docker/Keys.scala
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,10 @@ private[packager] trait DockerKeysEx extends DockerKeys {
lazy val dockerAdditionalPermissions =
taskKey[Seq[(DockerChmodType, String)]]("Explicit chmod calls to some of the paths.")
val dockerApiVersion = TaskKey[Option[DockerApiVersion]]("dockerApiVersion", "The docker server api version")
val dockerLayerGrouping = settingKey[String => Option[Int]](
"Group files by path into in layers to increase docker cache hits. " +
"Lower index means the file would be a part of an earlier layer."
)
val dockerLayerMappings =
taskKey[Seq[LayeredMapping]]("List of layer, source file and destination in Docker image.")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.typesafe.sbt.packager.docker

import java.io.File

/**
* Mapping of file to intermediate layers.
*
* @param layerId The identifier in the layer used to increase cache hits in
* docker caching. LayerId is present in docker:stage directory structure
* and in intermediate image produced in the multi-stage docker build.
* None means the layering is skipped for this file.
* @param file The file produced by universal/stage to be moved into `Docker / stage` directory.
* @param path The path in the final image
*/
case class LayeredMapping(layerId: Option[Int], file: File, path: String)
34 changes: 22 additions & 12 deletions src/sbt-test/docker/file-permission/build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,19 @@ lazy val root = (project in file("."))
assert(lines(2).substring(0, 25) == "LABEL snp-multi-stage-id=") // random generated id is hard to test
assertEquals(lines.drop(3),
"""WORKDIR /opt/docker
|COPY opt /opt
|COPY 1/opt /1/opt
|COPY 2/opt /2/opt
|USER root
|RUN ["chmod", "-R", "u=rX,g=rX", "/opt/docker"]
|RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
|RUN ["chmod", "-R", "u=rX,g=rX", "/1/opt/docker"]
|RUN ["chmod", "-R", "u=rX,g=rX", "/2/opt/docker"]
|RUN ["chmod", "u+x,g+x", "/1/opt/docker/bin/file-permission-test"]
|
|FROM fabric8/java-centos-openjdk8-jdk
|USER root
|RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
|WORKDIR /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /opt/docker /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /1/opt/docker /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /2/opt/docker /opt/docker
|USER 1001:0
|ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
|CMD []""".stripMargin.linesIterator.toList)
Expand All @@ -42,7 +45,8 @@ lazy val root = (project in file("."))
|USER root
|RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
|WORKDIR /opt/docker
|COPY opt /opt
|COPY 1/opt /opt
|COPY 2/opt /opt
|USER 1001:0
|ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
|CMD []""".stripMargin.linesIterator.toList)
Expand All @@ -56,7 +60,8 @@ lazy val root = (project in file("."))
|USER root
|RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 5000 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 5000 sbt || addgroup -g 5000 -S sbt )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 5000 demiourgos728 || adduser -S -u 1001 -G sbt demiourgos728 ))
|WORKDIR /opt/docker
|COPY opt /opt
|COPY 1/opt /opt
|COPY 2/opt /opt
|USER 1001:5000
|ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
|CMD []""".stripMargin.linesIterator.toList)
Expand All @@ -70,7 +75,8 @@ lazy val root = (project in file("."))
|USER root
|RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
|WORKDIR /opt/docker
|COPY opt /opt
|COPY 1/opt /opt
|COPY 2/opt /opt
|RUN ["chmod", "-R", "u=rX,g=rX", "/opt/docker"]
|RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
|USER 1001:0
Expand All @@ -84,7 +90,8 @@ lazy val root = (project in file("."))
assertEquals(lines,
"""FROM fabric8/java-centos-openjdk8-jdk
|WORKDIR /opt/docker
|COPY --chown=daemon:root opt /opt
|COPY --chown=daemon:root 1/opt /opt
|COPY --chown=daemon:root 2/opt /opt
|USER daemon
|ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
|CMD []""".stripMargin.linesIterator.toList)
Expand All @@ -99,16 +106,19 @@ lazy val root = (project in file("."))
assert(lines(2).substring(0, 25) == "LABEL snp-multi-stage-id=") // random generated id is hard to test
assertEquals(lines.drop(3),
"""WORKDIR /opt/docker
|COPY opt /opt
|COPY 1/opt /1/opt
|COPY 2/opt /2/opt
|USER root
|RUN ["chmod", "-R", "u=rwX,g=rwX", "/opt/docker"]
|RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
|RUN ["chmod", "-R", "u=rwX,g=rwX", "/1/opt/docker"]
|RUN ["chmod", "-R", "u=rwX,g=rwX", "/2/opt/docker"]
|RUN ["chmod", "u+x,g+x", "/1/opt/docker/bin/file-permission-test"]
|
|FROM fabric8/java-centos-openjdk8-jdk
|USER root
|RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
|WORKDIR /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /opt/docker /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /1/opt/docker /opt/docker
|COPY --from=stage0 --chown=demiourgos728:root /2/opt/docker /opt/docker
|USER 1001:0
|ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
|CMD []""".stripMargin.linesIterator.toList)
Expand Down
4 changes: 2 additions & 2 deletions src/sbt-test/docker/test-executableScriptName/test
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generate the Docker image locally
> docker:publishLocal
$ exists target/docker/stage/Dockerfile
$ exists target/docker/stage/opt/docker/bin/docker-exec
$ exists target/docker/stage/1/opt/docker/bin/docker-exec
> checkDockerfile
$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
10 changes: 10 additions & 0 deletions src/sbt-test/docker/test-layer-groups/build.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
enablePlugins(JavaAppPackaging)

organization := "com.example"
name := "docker-groups"
version := "0.1.0"

dockerPackageMappings in Docker ++= Seq(
(baseDirectory.value / "docker" / "spark-env.sh") -> "/opt/docker/spark/spark-env.sh",
(baseDirectory.value / "docker" / "log4j.properties") -> "/opt/docker/spark/log4j.properties"
)
1 change: 1 addition & 0 deletions src/sbt-test/docker/test-layer-groups/changes/nolayers.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dockerLayerGrouping in Docker := (_ => None)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
foo=goo
1 change: 1 addition & 0 deletions src/sbt-test/docker/test-layer-groups/docker/spark-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
echo "Hello!"
14 changes: 14 additions & 0 deletions src/sbt-test/docker/test-layer-groups/layers.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
dockerLayerGrouping in Docker := {
val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
(path: String) =>
{
val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
Some(2)
else if (pathInWorkdir.startsWith("/bin/"))
Some(123)
else if (pathInWorkdir.startsWith("/spark/"))
Some(54)
else None
}
}
1 change: 1 addition & 0 deletions src/sbt-test/docker/test-layer-groups/project/plugins.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
object Main extends App {
println("Hello world")
}
Loading

0 comments on commit 411bb5a

Please sign in to comment.