Merge pull request GoogleContainerTools#118 from nkubala/cache

add layer caching

Author: nkubala

cmd/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//differs:go_default_library",
+        "//pkg/cache:go_default_library",
         "//pkg/util:go_default_library",
         "//util:go_default_library",
         "//vendor/github.com/docker/docker/client:go_default_library",

cmd/root.go

@@ -20,10 +20,13 @@ import (
 	goflag "flag"
 	"fmt"
 	"os"
+	"os/user"
+	"path/filepath"
 	"sort"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/container-diff/differs"
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
 	"github.com/GoogleCloudPlatform/container-diff/util"
 	"github.com/docker/docker/client"
@@ -35,6 +38,7 @@ import (
 var json bool
 var save bool
 var types diffTypes
+var noCache bool
 
 var LogLevel string
 
@@ -120,25 +124,50 @@ func getPrepperForImage(image string) (pkgutil.Prepper, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	cacheDir, err := cacheDir()
+	if err != nil {
+		return nil, err
+	}
+	var fsCache cache.Cache
+	if !noCache {
+		fsCache, err = cache.NewFileCache(cacheDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if pkgutil.IsTar(image) {
 		return pkgutil.TarPrepper{
 			Source: image,
 			Client: cli,
+			Cache:  fsCache,
 		}, nil
 
 	} else if strings.HasPrefix(image, DaemonPrefix) {
 		return pkgutil.DaemonPrepper{
 			Source: strings.Replace(image, DaemonPrefix, "", -1),
 			Client: cli,
+			Cache:  fsCache,
 		}, nil
 	}
 	// either has remote prefix or has no prefix, in which case we force remote
 	return pkgutil.CloudPrepper{
 		Source: strings.Replace(image, RemotePrefix, "", -1),
 		Client: cli,
+		Cache:  fsCache,
 	}, nil
 }
 
+func cacheDir() (string, error) {
+	user, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+	rootDir := filepath.Join(user.HomeDir, ".container-diff")
+	return filepath.Join(rootDir, "cache"), nil
+}
+
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
 	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
@@ -175,4 +204,5 @@ func addSharedFlags(cmd *cobra.Command) {
 	cmd.Flags().VarP(&types, "type", "t", "This flag sets the list of analyzer types to use. Set it repeatedly to use multiple analyzers.")
 	cmd.Flags().BoolVarP(&save, "save", "s", false, "Set this flag to save rather than remove the final image filesystems on exit.")
 	cmd.Flags().BoolVarP(&util.SortSize, "order", "o", false, "Set this flag to sort any file/package results by descending size. Otherwise, they will be sorted by name.")
+	cmd.Flags().BoolVarP(&noCache, "no-cache", "n", false, "Set this to force retrieval of layers on each run.")
 }

pkg/cache/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cache.go",
+        "file_cache.go",
+    ],
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/sirupsen/logrus:go_default_library"],
+)

pkg/cache/cache.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2017 Google, Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import "io"
+
+type Cache interface {
+	HasLayer(string) bool
+	GetLayer(string) (io.ReadCloser, error)
+	SetLayer(string, io.Reader) (io.ReadCloser, error)
+	Invalidate(string) error
+}

pkg/cache/file_cache.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2017 Google, Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+type FileCache struct {
+	RootDir string
+}
+
+func NewFileCache(dir string) (*FileCache, error) {
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return nil, err
+	}
+	return &FileCache{RootDir: dir}, nil
+}
+
+func (c *FileCache) HasLayer(layerId string) bool {
+	_, err := os.Stat(filepath.Join(c.RootDir, layerId))
+	return !os.IsNotExist(err)
+}
+
+func (c *FileCache) SetLayer(layerId string, r io.Reader) (io.ReadCloser, error) {
+	fullpath := filepath.Join(c.RootDir, layerId)
+	entry, err := os.Create(fullpath)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := io.Copy(entry, r); err != nil {
+		return nil, err
+	}
+	return c.GetLayer(layerId)
+}
+
+func (c *FileCache) GetLayer(layerId string) (io.ReadCloser, error) {
+	logrus.Infof("retrieving layer %s from cache", layerId)
+	return os.Open(filepath.Join(c.RootDir, layerId))
+}
+
+func (c *FileCache) Invalidate(layerId string) error {
+	return os.RemoveAll(filepath.Join(c.RootDir, layerId))
+}

pkg/util/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/cache:go_default_library",
         "//vendor/github.com/containers/image/docker:go_default_library",
         "//vendor/github.com/containers/image/docker/daemon:go_default_library",
         "//vendor/github.com/containers/image/docker/tarfile:go_default_library",

pkg/util/cloud_prepper.go

@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/containers/image/docker"
 	"github.com/docker/docker/client"
 )
@@ -25,6 +26,7 @@ import (
 type CloudPrepper struct {
 	Source string
 	Client *client.Client
+	Cache  cache.Cache
 }
 
 func (p CloudPrepper) Name() string {
@@ -45,7 +47,7 @@ func (p CloudPrepper) GetFileSystem() (string, error) {
 		return "", err
 	}
 
-	return getFileSystemFromReference(ref, p.Source)
+	return getFileSystemFromReference(ref, p.Source, p.Cache)
 }
 
 func (p CloudPrepper) GetConfig() (ConfigSchema, error) {

pkg/util/daemon_prepper.go

@@ -19,6 +19,7 @@ package util
 import (
 	"context"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/containers/image/docker/daemon"
 
 	"github.com/docker/docker/client"
@@ -28,6 +29,7 @@ import (
 type DaemonPrepper struct {
 	Source string
 	Client *client.Client
+	Cache  cache.Cache
 }
 
 func (p DaemonPrepper) Name() string {
@@ -47,7 +49,7 @@ func (p DaemonPrepper) GetFileSystem() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return getFileSystemFromReference(ref, p.Source)
+	return getFileSystemFromReference(ref, p.Source, p.Cache)
 }
 
 func (p DaemonPrepper) GetConfig() (ConfigSchema, error) {

pkg/util/image_prep_utils.go

@@ -21,10 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"strings"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/types"
 	"github.com/sirupsen/logrus"
@@ -91,7 +93,7 @@ func getImageFromTar(tarPath string) (string, error) {
 	return tempPath, unpackDockerSave(tarPath, tempPath)
 }
 
-func getFileSystemFromReference(ref types.ImageReference, imageName string) (string, error) {
+func getFileSystemFromReference(ref types.ImageReference, imageName string, cache cache.Cache) (string, error) {
 	sanitizedName := strings.Replace(imageName, ":", "", -1)
 	sanitizedName = strings.Replace(sanitizedName, "/", "", -1)
 
@@ -113,10 +115,33 @@ func getFileSystemFromReference(ref types.ImageReference, imageName string) (str
 		return "", err
 	}
 
+	var bi io.ReadCloser
 	for _, b := range img.LayerInfos() {
-		bi, _, err := imgSrc.GetBlob(b)
+		layerId := b.Digest.String()
+		if cache == nil {
+			bi, _, err = imgSrc.GetBlob(b)
+			if err != nil {
+				logrus.Errorf("Failed to pull image layer: %s", err)
+				return "", err
+			}
+		} else if cache.HasLayer(layerId) {
+			logrus.Infof("cache hit for layer %s", layerId)
+			bi, err = cache.GetLayer(layerId)
+		} else {
+			logrus.Infof("cache miss for layer %s", layerId)
+			bi, _, err = imgSrc.GetBlob(b)
+			if err != nil {
+				logrus.Errorf("Failed to pull image layer: %s", err)
+				return "", err
+			}
+			bi, err = cache.SetLayer(layerId, bi)
+			if err != nil {
+				logrus.Errorf("error when caching layer %s: %s", layerId, err)
+				cache.Invalidate(layerId)
+			}
+		}
 		if err != nil {
-			logrus.Errorf("Failed to pull image layer: %s", err)
+			logrus.Errorf("Failed to retrieve image layer: %s", err)
 			return "", err
 		}
 		// try and detect layer compression

pkg/util/tar_prepper.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/containers/image/docker/tarfile"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
@@ -31,6 +32,7 @@ import (
 type TarPrepper struct {
 	Source string
 	Client *client.Client
+	Cache  cache.Cache
 }
 
 func (p TarPrepper) Name() string {

util/BUILD.bazel

@@ -24,11 +24,15 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "file_cache_test.go",
         "fs_utils_test.go",
         "output_sort_utils_test.go",
         "package_diff_utils_test.go",
         "tar_utils_test.go",
     ],
     library = ":go_default_library",
-    deps = ["//pkg/util:go_default_library"],
+    deps = [
+        "//pkg/cache:go_default_library",
+        "//pkg/util:go_default_library",
+    ],
 )