diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 82d597778ea..b1af3d86515 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -559,19 +559,39 @@ func main() { }, &cli.StringFlag{ - Name: "backend-type", + Name: "source-backend-type", Value: "", - Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3'", + Usage: "Type of storage backend, possible values: 'oss', 's3'", EnvVars: []string{"BACKEND_TYPE"}, }, &cli.StringFlag{ - Name: "backend-config", + Name: "source-backend-config", Value: "", - Usage: "Json string for storage backend configuration", + Usage: "Json configuration string for storage backend", EnvVars: []string{"BACKEND_CONFIG"}, }, &cli.PathFlag{ - Name: "backend-config-file", + Name: "source-backend-config-file", + Value: "", + TakesFile: true, + Usage: "Json configuration file for storage backend", + EnvVars: []string{"BACKEND_CONFIG_FILE"}, + }, + + &cli.StringFlag{ + Name: "target-backend-type", + Value: "", + Usage: "Type of storage backend, possible values: 'oss', 's3'", + EnvVars: []string{"BACKEND_TYPE"}, + }, + &cli.StringFlag{ + Name: "target-backend-config", + Value: "", + Usage: "Json configuration string for storage backend", + EnvVars: []string{"BACKEND_CONFIG"}, + }, + &cli.PathFlag{ + Name: "target-backend-config-file", Value: "", TakesFile: true, Usage: "Json configuration file for storage backend", @@ -612,7 +632,12 @@ func main() { Action: func(c *cli.Context) error { setupLogLevel(c) - backendType, backendConfig, err := getBackendConfig(c, "", false) + sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false) + if err != nil { + return err + } + + targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false) if err != nil { return err } @@ -623,16 +648,20 @@ func main() { } checker, err := checker.New(checker.Opt{ - WorkDir: c.String("work-dir"), - Source: c.String("source"), - Target: c.String("target"), + WorkDir: c.String("work-dir"), + + Source: c.String("source"), + Target: c.String("target"), + SourceInsecure: c.Bool("source-insecure"), + TargetInsecure: c.Bool("target-insecure"), + SourceBackendType: sourceBackendType, + SourceBackendConfig: sourceBackendConfig, + TargetBackendType: targetBackendType, + TargetBackendConfig: targetBackendConfig, + MultiPlatform: c.Bool("multi-platform"), - SourceInsecure: c.Bool("source-insecure"), - TargetInsecure: c.Bool("target-insecure"), NydusImagePath: c.String("nydus-image"), NydusdPath: c.String("nydusd"), - BackendType: backendType, - BackendConfig: backendConfig, ExpectedArch: arch, }) if err != nil { diff --git a/contrib/nydusify/pkg/checker/checker.go b/contrib/nydusify/pkg/checker/checker.go index 58dcf92e05e..b6cbd2eaf06 100644 --- a/contrib/nydusify/pkg/checker/checker.go +++ b/contrib/nydusify/pkg/checker/checker.go @@ -13,43 +13,44 @@ import ( "github.com/sirupsen/logrus" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" ) // Opt defines Checker options. -// Note: target is the Nydus image reference. +// Note: target is the nydus image reference. type Opt struct { - WorkDir string - Source string - Target string - SourceInsecure bool - TargetInsecure bool + WorkDir string + + Source string + Target string + SourceInsecure bool + TargetInsecure bool + SourceBackendType string + SourceBackendConfig string + TargetBackendType string + TargetBackendConfig string + MultiPlatform bool NydusImagePath string NydusdPath string - BackendType string - BackendConfig string ExpectedArch string } -// Checker validates Nydus image manifest, bootstrap and mounts filesystem -// by Nydusd to compare file metadata and data with OCI image. +// Checker validates nydus image manifest, bootstrap and mounts filesystem +// by nydusd to compare file metadata and data between OCI / nydus image. type Checker struct { Opt sourceParser *parser.Parser targetParser *parser.Parser } -// New creates Checker instance, target is the Nydus image reference. +// New creates Checker instance, target is the nydus image reference. func New(opt Opt) (*Checker, error) { - // TODO: support source and target resolver targetRemote, err := provider.DefaultRemote(opt.Target, opt.TargetInsecure) if err != nil { - return nil, errors.Wrap(err, "Init target image parser") + return nil, errors.Wrap(err, "init target image parser") } targetParser, err := parser.New(targetRemote, opt.ExpectedArch) if err != nil { @@ -63,7 +64,7 @@ func New(opt Opt) (*Checker, error) { return nil, errors.Wrap(err, "Init source image parser") } sourceParser, err = parser.New(sourceRemote, opt.ExpectedArch) - if sourceParser == nil { + if err != nil { return nil, errors.Wrap(err, "failed to create parser") } } @@ -77,7 +78,7 @@ func New(opt Opt) (*Checker, error) { return checker, nil } -// Check checks Nydus image, and outputs image information to work +// Check checks nydus image, and outputs image information to work // directory, the check workflow is composed of various rules. func (checker *Checker) Check(ctx context.Context) error { if err := checker.check(ctx); err != nil { @@ -93,12 +94,13 @@ func (checker *Checker) Check(ctx context.Context) error { return nil } -// Check checks Nydus image, and outputs image information to work +// Check checks nydus image, and outputs image information to work // directory, the check workflow is composed of various rules. func (checker *Checker) check(ctx context.Context) error { + logrus.WithField("image", checker.targetParser.Remote.Ref).Infof("parsing image") targetParsed, err := checker.targetParser.Parse(ctx) if err != nil { - return errors.Wrap(err, "parse Nydus image") + return errors.Wrap(err, "parse nydus image") } var sourceParsed *parser.Parsed @@ -107,89 +109,66 @@ func (checker *Checker) check(ctx context.Context) error { if err != nil { return errors.Wrap(err, "parse source image") } - } else { - sourceParsed = targetParsed } if err := os.RemoveAll(checker.WorkDir); err != nil { return errors.Wrap(err, "clean up work directory") } - if err := os.MkdirAll(filepath.Join(checker.WorkDir, "fs"), 0755); err != nil { - return errors.Wrap(err, "create work directory") - } - - if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil { - return errors.Wrap(err, "output image information") - } - - mode := "direct" - digestValidate := false - if targetParsed.NydusImage != nil { - nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest) - if nydusManifest != nil { - v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5) - if v == utils.V5 { - // Digest validate is not currently supported for v6, - // but v5 supports it. In order to make the check more sufficient, - // this validate needs to be turned on for v5. - digestValidate = true - } + if sourceParsed != nil { + if err := checker.Output(ctx, sourceParsed, filepath.Join(checker.WorkDir, "source")); err != nil { + return errors.Wrapf(err, "output image information: %s", sourceParsed.Remote.Ref) } } - var sourceRemote *remote.Remote - if checker.sourceParser != nil { - sourceRemote = checker.sourceParser.Remote + if targetParsed != nil { + if err := checker.Output(ctx, targetParsed, filepath.Join(checker.WorkDir, "target")); err != nil { + return errors.Wrapf(err, "output image information: %s", targetParsed.Remote.Ref) + } } rules := []rule.Rule{ &rule.ManifestRule{ - SourceParsed: sourceParsed, - TargetParsed: targetParsed, - MultiPlatform: checker.MultiPlatform, - BackendType: checker.BackendType, - ExpectedArch: checker.ExpectedArch, + SourceParsed: sourceParsed, + TargetParsed: targetParsed, }, &rule.BootstrapRule{ - Parsed: targetParsed, - NydusImagePath: checker.NydusImagePath, - BackendType: checker.BackendType, - BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"), - DebugOutputPath: filepath.Join(checker.WorkDir, "nydus_bootstrap_debug.json"), + WorkDir: checker.WorkDir, + NydusImagePath: checker.NydusImagePath, + + SourceParsed: sourceParsed, + TargetParsed: targetParsed, + SourceBackendType: checker.SourceBackendType, + SourceBackendConfig: checker.SourceBackendConfig, + TargetBackendType: checker.TargetBackendType, + TargetBackendConfig: checker.TargetBackendConfig, }, &rule.FilesystemRule{ - Source: checker.Source, - SourceMountPath: filepath.Join(checker.WorkDir, "fs/source_mounted"), - SourceParsed: sourceParsed, - SourcePath: filepath.Join(checker.WorkDir, "fs/source"), - SourceRemote: sourceRemote, - Target: checker.Target, - TargetInsecure: checker.TargetInsecure, - PlainHTTP: checker.targetParser.Remote.IsWithHTTP(), - NydusdConfig: tool.NydusdConfig{ - EnablePrefetch: true, - NydusdPath: checker.NydusdPath, - BackendType: checker.BackendType, - BackendConfig: checker.BackendConfig, - BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"), - ConfigPath: filepath.Join(checker.WorkDir, "fs/nydusd_config.json"), - BlobCacheDir: filepath.Join(checker.WorkDir, "fs/nydus_blobs"), - MountPath: filepath.Join(checker.WorkDir, "fs/nydus_mounted"), - APISockPath: filepath.Join(checker.WorkDir, "fs/nydus_api.sock"), - Mode: mode, - DigestValidate: digestValidate, + WorkDir: checker.WorkDir, + NydusdPath: checker.NydusdPath, + + SourceImage: &rule.Image{ + Parsed: sourceParsed, + Insecure: checker.SourceInsecure, + }, + TargetImage: &rule.Image{ + Parsed: targetParsed, + Insecure: checker.TargetInsecure, }, + SourceBackendType: checker.SourceBackendType, + SourceBackendConfig: checker.SourceBackendConfig, + TargetBackendType: checker.TargetBackendType, + TargetBackendConfig: checker.TargetBackendConfig, }, } for _, rule := range rules { if err := rule.Validate(); err != nil { - return errors.Wrapf(err, "validate rule %s", rule.Name()) + return errors.Wrapf(err, "validate %s failed", rule.Name()) } } - logrus.Infof("Verified Nydus image %s", checker.targetParser.Remote.Ref) + logrus.Info("verified image") return nil } diff --git a/contrib/nydusify/pkg/checker/output.go b/contrib/nydusify/pkg/checker/output.go index a9678a008c4..c68f11eff70 100644 --- a/contrib/nydusify/pkg/checker/output.go +++ b/contrib/nydusify/pkg/checker/output.go @@ -7,12 +7,16 @@ package checker import ( "context" "encoding/json" + "io" "os" "path/filepath" + "github.com/containerd/containerd/archive/compression" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" ) @@ -25,70 +29,96 @@ func prettyDump(obj interface{}, name string) error { return os.WriteFile(name, bytes, 0644) } -// Output outputs OCI and Nydus image manifest, index, config to JSON file. +// Output outputs OCI and nydus image manifest, index, config to JSON file. // Prefer to use source image to output OCI image information. func (checker *Checker) Output( - ctx context.Context, sourceParsed, targetParsed *parser.Parsed, outputPath string, + ctx context.Context, parsed *parser.Parsed, dir string, ) error { - logrus.Infof("Dumping OCI and Nydus manifests to %s", outputPath) + logrus.WithField("type", tool.CheckImageType(parsed)).WithField("image", parsed.Remote.Ref).Info("dumping manifest") - if sourceParsed.Index != nil { + if err := os.MkdirAll(dir, 0755); err != nil { + return errors.Wrap(err, "create output directory") + } + + if parsed.Index != nil && parsed.OCIImage != nil { if err := prettyDump( - sourceParsed.Index, - filepath.Join(outputPath, "oci_index.json"), + parsed.Index, + filepath.Join(dir, "oci_index.json"), ); err != nil { return errors.Wrap(err, "output oci index file") } } - if targetParsed.Index != nil { + if parsed.Index != nil && parsed.NydusImage != nil { if err := prettyDump( - targetParsed.Index, - filepath.Join(outputPath, "nydus_index.json"), + parsed.Index, + filepath.Join(dir, "nydus_index.json"), ); err != nil { return errors.Wrap(err, "output nydus index file") } } - if sourceParsed.OCIImage != nil { + if parsed.OCIImage != nil { if err := prettyDump( - sourceParsed.OCIImage.Manifest, - filepath.Join(outputPath, "oci_manifest.json"), + parsed.OCIImage.Manifest, + filepath.Join(dir, "oci_manifest.json"), ); err != nil { return errors.Wrap(err, "output OCI manifest file") } if err := prettyDump( - sourceParsed.OCIImage.Config, - filepath.Join(outputPath, "oci_config.json"), + parsed.OCIImage.Config, + filepath.Join(dir, "oci_config.json"), ); err != nil { return errors.Wrap(err, "output OCI config file") } } - if targetParsed.NydusImage != nil { + if parsed.NydusImage != nil { if err := prettyDump( - targetParsed.NydusImage.Manifest, - filepath.Join(outputPath, "nydus_manifest.json"), + parsed.NydusImage.Manifest, + filepath.Join(dir, "nydus_manifest.json"), ); err != nil { - return errors.Wrap(err, "output Nydus manifest file") + return errors.Wrap(err, "output nydus manifest file") } if err := prettyDump( - targetParsed.NydusImage.Config, - filepath.Join(outputPath, "nydus_config.json"), + parsed.NydusImage.Config, + filepath.Join(dir, "nydus_config.json"), ); err != nil { - return errors.Wrap(err, "output Nydus config file") + return errors.Wrap(err, "output nydus config file") } - target := filepath.Join(outputPath, "nydus_bootstrap") - logrus.Infof("Pulling Nydus bootstrap to %s", target) - bootstrapReader, err := checker.targetParser.PullNydusBootstrap(ctx, targetParsed.NydusImage) + bootstrapDir := filepath.Join(dir, "nydus_bootstrap") + logrus.WithField("type", tool.CheckImageType(parsed)).WithField("image", parsed.Remote.Ref).Info("pulling bootstrap") + var parser *parser.Parser + if dir == "source" { + parser = checker.sourceParser + } else { + parser = checker.targetParser + } + bootstrapReader, err := parser.PullNydusBootstrap(ctx, parsed.NydusImage) if err != nil { - return errors.Wrap(err, "pull Nydus bootstrap layer") + return errors.Wrap(err, "pull nydus bootstrap layer") } defer bootstrapReader.Close() - if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, target); err != nil { - return errors.Wrap(err, "unpack Nydus bootstrap layer") + tarRc, err := compression.DecompressStream(bootstrapReader) + if err != nil { + return err + } + defer tarRc.Close() + + diffID := digest.SHA256.Digester() + if err := utils.UnpackFromTar(io.TeeReader(tarRc, diffID.Hash()), bootstrapDir); err != nil { + return errors.Wrap(err, "unpack nydus bootstrap layer") + } + + diffIDs := parsed.NydusImage.Config.RootFS.DiffIDs + if diffIDs[len(diffIDs)-1] != diffID.Digest() { + return errors.Errorf( + "invalid bootstrap layer diff id: %s (calculated) != %s (in image config)", + diffID.Digest().String(), + diffIDs[len(diffIDs)-1].String(), + ) } } diff --git a/contrib/nydusify/pkg/checker/rule/bootstrap.go b/contrib/nydusify/pkg/checker/rule/bootstrap.go index 5adbeab543c..8cf90729cda 100644 --- a/contrib/nydusify/pkg/checker/rule/bootstrap.go +++ b/contrib/nydusify/pkg/checker/rule/bootstrap.go @@ -8,40 +8,53 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" ) -// BootstrapRule validates bootstrap in Nydus image +// BootstrapRule validates bootstrap in nydus image type BootstrapRule struct { - Parsed *parser.Parsed - BootstrapPath string - NydusImagePath string - DebugOutputPath string - BackendType string + WorkDir string + NydusImagePath string + + SourceParsed *parser.Parsed + TargetParsed *parser.Parsed + SourceBackendType string + SourceBackendConfig string + TargetBackendType string + TargetBackendConfig string } -type bootstrapDebug struct { +type output struct { Blobs []string `json:"blobs"` } func (rule *BootstrapRule) Name() string { - return "Bootstrap" + return "bootstrap" } -func (rule *BootstrapRule) Validate() error { - logrus.Infof("Checking Nydus bootstrap") +func (rule *BootstrapRule) validate(parsed *parser.Parsed, dir, backendType string) error { + if parsed == nil || parsed.NydusImage == nil { + return nil + } + + logrus.WithField("type", tool.CheckImageType(parsed)).WithField("image", parsed.Remote.Ref).Info("checking bootstrap") + + bootstrapDir := filepath.Join(rule.WorkDir, dir, "nydus_bootstrap") + outputPath := filepath.Join(rule.WorkDir, dir, "nydus_output.json") // Get blob list in the blob table of bootstrap by calling // `nydus-image check` command builder := tool.NewBuilder(rule.NydusImagePath) if err := builder.Check(tool.BuilderOption{ - BootstrapPath: rule.BootstrapPath, - DebugOutputPath: rule.DebugOutputPath, + BootstrapPath: filepath.Join(bootstrapDir, utils.BootstrapFileNameInLayer), + DebugOutputPath: outputPath, }); err != nil { return errors.Wrap(err, "invalid nydus bootstrap format") } @@ -49,13 +62,13 @@ func (rule *BootstrapRule) Validate() error { // For registry garbage collection, nydus puts the blobs to // the layers in manifest, so here only need to check blob // list consistency for registry backend. - if rule.BackendType != "registry" { + if backendType != "registry" { return nil } - // Parse blob list from blob layers in Nydus manifest + // Parse blob list from blob layers in nydus manifest blobListInLayer := map[string]bool{} - layers := rule.Parsed.NydusImage.Manifest.Layers + layers := parsed.NydusImage.Manifest.Layers for i, layer := range layers { if i != len(layers)-1 { blobListInLayer[layer.Digest.Hex()] = true @@ -63,17 +76,17 @@ func (rule *BootstrapRule) Validate() error { } // Parse blob list from blob table of bootstrap - var bootstrap bootstrapDebug - bootstrapBytes, err := os.ReadFile(rule.DebugOutputPath) + var out output + outputBytes, err := os.ReadFile(outputPath) if err != nil { return errors.Wrap(err, "read bootstrap debug json") } - if err := json.Unmarshal(bootstrapBytes, &bootstrap); err != nil { + if err := json.Unmarshal(outputBytes, &out); err != nil { return errors.Wrap(err, "unmarshal bootstrap output JSON") } blobListInBootstrap := map[string]bool{} lostInLayer := false - for _, blobID := range bootstrap.Blobs { + for _, blobID := range out.Blobs { blobListInBootstrap[blobID] = true if !blobListInLayer[blobID] { lostInLayer = true @@ -94,3 +107,15 @@ func (rule *BootstrapRule) Validate() error { blobListInLayer, ) } + +func (rule *BootstrapRule) Validate() error { + if err := rule.validate(rule.SourceParsed, "source", rule.SourceBackendType); err != nil { + return errors.Wrap(err, "source image: invalid nydus bootstrap") + } + + if err := rule.validate(rule.TargetParsed, "target", rule.TargetBackendType); err != nil { + return errors.Wrap(err, "target image: invalid nydus bootstrap") + } + + return nil +} diff --git a/contrib/nydusify/pkg/checker/rule/filesystem.go b/contrib/nydusify/pkg/checker/rule/filesystem.go index 898251c971a..d27b42827d1 100644 --- a/contrib/nydusify/pkg/checker/rule/filesystem.go +++ b/contrib/nydusify/pkg/checker/rule/filesystem.go @@ -18,7 +18,6 @@ import ( "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" "github.com/pkg/errors" "github.com/pkg/xattr" @@ -29,18 +28,23 @@ import ( var WorkerCount uint = 8 // FilesystemRule compares file metadata and data in the two mountpoints: -// Mounted by Nydusd for Nydus image, +// Mounted by nydusd for nydus image, // Mounted by Overlayfs for OCI image. type FilesystemRule struct { - NydusdConfig tool.NydusdConfig - Source string - SourceMountPath string - SourceParsed *parser.Parsed - SourcePath string - SourceRemote *remote.Remote - Target string - TargetInsecure bool - PlainHTTP bool + WorkDir string + NydusdPath string + + SourceImage *Image + TargetImage *Image + SourceBackendType string + SourceBackendConfig string + TargetBackendType string + TargetBackendConfig string +} + +type Image struct { + Parsed *parser.Parsed + Insecure bool } // Node records file metadata and file data hash. @@ -66,14 +70,14 @@ type RegistryBackendConfig struct { func (node *Node) String() string { return fmt.Sprintf( - "Path: %s, Size: %d, Mode: %d, Rdev: %d, Symink: %s, UID: %d, GID: %d, "+ - "Xattrs: %v, Hash: %s", node.Path, node.Size, node.Mode, node.Rdev, node.Symlink, + "path: %s, size: %d, mode: %d, rdev: %d, symink: %s, uid: %d, gid: %d, "+ + "xattrs: %v, hash: %s", node.Path, node.Size, node.Mode, node.Rdev, node.Symlink, node.UID, node.GID, node.Xattrs, hex.EncodeToString(node.Hash), ) } func (rule *FilesystemRule) Name() string { - return "Filesystem" + return "filesystem" } func getXattrs(path string) (map[string][]byte, error) { @@ -132,13 +136,13 @@ func (rule *FilesystemRule) walk(rootfs string) (map[string]Node, error) { xattrs, err := getXattrs(path) if err != nil { - logrus.Warnf("Failed to get xattr: %s", err) + logrus.Warnf("failed to get xattr: %s", err) } // Calculate file data hash if the `backend-type` option be specified, // this will cause that nydusd read data from backend, it's network load var hash []byte - if rule.NydusdConfig.BackendType != "" && info.Mode().IsRegular() { + if info.Mode().IsRegular() { hash, err = utils.HashFile(path) if err != nil { return err @@ -166,89 +170,75 @@ func (rule *FilesystemRule) walk(rootfs string) (map[string]Node, error) { return nodes, nil } -func (rule *FilesystemRule) pullSourceImage() (*tool.Image, error) { - layers := rule.SourceParsed.OCIImage.Manifest.Layers - worker := utils.NewWorkerPool(WorkerCount, uint(len(layers))) - - for idx := range layers { - worker.Put(func(idx int) func() error { - return func() error { - layer := layers[idx] - reader, err := rule.SourceRemote.Pull(context.Background(), layer, true) - if err != nil { - return errors.Wrap(err, "pull source image layers from the remote registry") - } - - if err = utils.UnpackTargz(context.Background(), filepath.Join(rule.SourcePath, fmt.Sprintf("layer-%d", idx)), reader, true); err != nil { - return errors.Wrap(err, "unpack source image layers") - } - - return nil +func (rule *FilesystemRule) mountNydusImage(image *Image, dir string) (func() error, error) { + logrus.WithField("type", tool.CheckImageType(image.Parsed)).WithField("image", image.Parsed.Remote.Ref).Info("mounting image") + + digestValidate := false + if image.Parsed.NydusImage != nil { + nydusManifest := parser.FindNydusBootstrapDesc(&image.Parsed.NydusImage.Manifest) + if nydusManifest != nil { + v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5) + if v == utils.V5 { + // Digest validate is not currently supported for v6, + // but v5 supports it. In order to make the check more sufficient, + // this validate needs to be turned on for v5. + digestValidate = true } - }(idx)) - } - - if err := <-worker.Waiter(); err != nil { - return nil, errors.Wrap(err, "pull source image layers in wait") - } - - return &tool.Image{ - Layers: layers, - Source: rule.Source, - SourcePath: rule.SourcePath, - Rootfs: rule.SourceMountPath, - }, nil -} - -func (rule *FilesystemRule) mountSourceImage() (*tool.Image, error) { - logrus.Infof("Mounting source image to %s", rule.SourceMountPath) - - image, err := rule.pullSourceImage() - if err != nil { - return nil, errors.Wrap(err, "pull source image") + } } - if err := image.Umount(); err != nil { - return nil, errors.Wrap(err, "umount previous rootfs") + backendType := rule.SourceBackendType + backendConfig := rule.SourceBackendConfig + if dir == "target" { + backendType = rule.TargetBackendType + backendConfig = rule.TargetBackendConfig } - if err := image.Mount(); err != nil { - return nil, errors.Wrap(err, "mount source image") + blobCacheDir := filepath.Join(rule.WorkDir, dir, "nydus_blobs") + mountDir := filepath.Join(rule.WorkDir, dir, "mnt") + apiSockPath := filepath.Join(rule.WorkDir, dir, "nydus_api.sock") + + nydusdConfig := tool.NydusdConfig{ + EnablePrefetch: true, + NydusdPath: rule.NydusdPath, + BackendType: backendType, + BackendConfig: backendConfig, + BootstrapPath: filepath.Join(rule.WorkDir, dir, "nydus_bootstrap/image/image.boot"), + ConfigPath: filepath.Join(rule.WorkDir, dir, "nydus_config.json"), + BlobCacheDir: blobCacheDir, + APISockPath: apiSockPath, + MountPath: mountDir, + Mode: "direct", + DigestValidate: digestValidate, } - return image, nil -} - -func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { - logrus.Infof("Mounting Nydus image to %s", rule.NydusdConfig.MountPath) - - if err := os.MkdirAll(rule.NydusdConfig.BlobCacheDir, 0755); err != nil { - return nil, errors.Wrap(err, "create blob cache directory for Nydusd") + if err := os.MkdirAll(nydusdConfig.BlobCacheDir, 0755); err != nil { + return nil, errors.Wrap(err, "create blob cache directory for nydusd") } - if err := os.MkdirAll(rule.NydusdConfig.MountPath, 0755); err != nil { - return nil, errors.Wrap(err, "create mountpoint directory of Nydus image") + if err := os.MkdirAll(nydusdConfig.MountPath, 0755); err != nil { + return nil, errors.Wrap(err, "create mountpoint directory of nydus image") } - parsed, err := reference.ParseNormalizedNamed(rule.Target) + ref, err := reference.ParseNormalizedNamed(image.Parsed.Remote.Ref) if err != nil { return nil, err } - if rule.NydusdConfig.BackendType == "" { - rule.NydusdConfig.BackendType = "registry" + if nydusdConfig.BackendType == "" { + nydusdConfig.BackendType = "registry" - if rule.NydusdConfig.BackendConfig == "" { - backendConfig, err := utils.NewRegistryBackendConfig(parsed, rule.TargetInsecure) + if nydusdConfig.BackendConfig == "" { + backendConfig, err := utils.NewRegistryBackendConfig(ref, image.Insecure) if err != nil { return nil, errors.Wrap(err, "failed to parse backend configuration") } - if rule.TargetInsecure { + if image.Insecure { backendConfig.SkipVerify = true } - if rule.PlainHTTP { + if image.Parsed.Remote.IsWithHTTP() { backendConfig.Scheme = "http" } @@ -256,38 +246,129 @@ func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { if err != nil { return nil, errors.Wrap(err, "parse registry backend config") } - rule.NydusdConfig.BackendConfig = string(bytes) + nydusdConfig.BackendConfig = string(bytes) } } - nydusd, err := tool.NewNydusd(rule.NydusdConfig) + nydusd, err := tool.NewNydusd(nydusdConfig) if err != nil { - return nil, errors.Wrap(err, "create Nydusd daemon") + return nil, errors.Wrap(err, "create nydusd daemon") } if err := nydusd.Mount(); err != nil { - return nil, errors.Wrap(err, "mount Nydus image") + return nil, errors.Wrap(err, "mount nydus image") + } + + umount := func() error { + if err := nydusd.Umount(false); err != nil { + return errors.Wrap(err, "umount nydus image") + } + if err := os.RemoveAll(blobCacheDir); err != nil { + logrus.WithError(err).Warnf("cleanup blob cache directory: %s", blobCacheDir) + } + if err := os.RemoveAll(mountDir); err != nil { + logrus.WithError(err).Warnf("cleanup mount directory: %s", mountDir) + } + if err := os.RemoveAll(apiSockPath); err != nil { + logrus.WithError(err).Warnf("cleanup api sock file: %s", apiSockPath) + } + return nil + } + + return umount, nil +} + +func (rule *FilesystemRule) mountOCIImage(image *Image, dir string) (func() error, error) { + logrus.WithField("type", tool.CheckImageType(image.Parsed)).WithField("image", image.Parsed.Remote.Ref).Infof("mounting image") + + mountPath := filepath.Join(rule.WorkDir, dir, "mnt") + if err := os.MkdirAll(mountPath, 0755); err != nil { + return nil, errors.Wrap(err, "create mountpoint directory") + } + layerBasePath := filepath.Join(rule.WorkDir, dir, "layers") + if err := os.MkdirAll(layerBasePath, 0755); err != nil { + return nil, errors.Wrap(err, "create layer base directory") } - return nydusd, nil + layers := image.Parsed.OCIImage.Manifest.Layers + worker := utils.NewWorkerPool(WorkerCount, uint(len(layers))) + + for idx := range layers { + worker.Put(func(idx int) func() error { + return func() error { + layer := layers[idx] + reader, err := image.Parsed.Remote.Pull(context.Background(), layer, true) + if err != nil { + return errors.Wrap(err, "pull source image layers from the remote registry") + } + + layerDir := filepath.Join(layerBasePath, fmt.Sprintf("layer-%d", idx)) + if err = utils.UnpackTargz(context.Background(), layerDir, reader, true); err != nil { + return errors.Wrap(err, "unpack source image layers") + } + + return nil + } + }(idx)) + } + + if err := <-worker.Waiter(); err != nil { + return nil, errors.Wrap(err, "pull source image layers in wait") + } + + mounter := &tool.Image{ + Layers: layers, + LayerBaseDir: layerBasePath, + Rootfs: mountPath, + } + + if err := mounter.Umount(); err != nil { + return nil, errors.Wrap(err, "umount previous rootfs") + } + + if err := mounter.Mount(); err != nil { + return nil, errors.Wrap(err, "mount source image") + } + + umount := func() error { + if err := mounter.Umount(); err != nil { + logrus.WithError(err).Warnf("umount rootfs") + } + if err := os.RemoveAll(layerBasePath); err != nil { + logrus.WithError(err).Warnf("cleanup layers directory %s", layerBasePath) + } + return nil + } + + return umount, nil } -func (rule *FilesystemRule) verify() error { - logrus.Infof("Verifying filesystem for source and Nydus image") +func (rule *FilesystemRule) mountImage(image *Image, dir string) (func() error, error) { + if image.Parsed.OCIImage != nil { + return rule.mountOCIImage(image, dir) + } else if image.Parsed.NydusImage != nil { + return rule.mountNydusImage(image, dir) + } + + return nil, fmt.Errorf("invalid image for mounting") +} + +func (rule *FilesystemRule) verify(sourceRootfs, targetRootfs string) error { + logrus.Infof("comparing filesystem") sourceNodes := map[string]Node{} - // Concurrently walk the rootfs directory of source and Nydus image + // Concurrently walk the rootfs directory of source and nydus image walkErr := make(chan error) go func() { var err error - sourceNodes, err = rule.walk(rule.SourceMountPath) + sourceNodes, err = rule.walk(sourceRootfs) walkErr <- err }() - nydusNodes, err := rule.walk(rule.NydusdConfig.MountPath) + targetNodes, err := rule.walk(targetRootfs) if err != nil { - return errors.Wrap(err, "walk rootfs of Nydus image") + return errors.Wrap(err, "walk rootfs of source image") } if err := <-walkErr; err != nil { @@ -295,54 +376,44 @@ func (rule *FilesystemRule) verify() error { } for path, sourceNode := range sourceNodes { - nydusNode, exist := nydusNodes[path] + targetNode, exist := targetNodes[path] if !exist { - return fmt.Errorf("File not found in Nydus image: %s", path) + return fmt.Errorf("file not found in target image: %s", path) } - delete(nydusNodes, path) + delete(targetNodes, path) - if path != "/" && !reflect.DeepEqual(sourceNode, nydusNode) { - return fmt.Errorf("File not match in Nydus image: %s <=> %s", sourceNode.String(), nydusNode.String()) + if path != "/" && !reflect.DeepEqual(sourceNode, targetNode) { + return fmt.Errorf("file not match in target image:\n\t[source] %s\n\t[target] %s", sourceNode.String(), targetNode.String()) } } - for path := range nydusNodes { - return fmt.Errorf("File not found in source image: %s", path) + for path := range targetNodes { + return fmt.Errorf("file not found in source image: %s", path) } return nil } func (rule *FilesystemRule) Validate() error { - // Skip filesystem validation if no source image be specified - if rule.Source == "" { + // Skip filesystem validation if no source or target image be specified + if rule.SourceImage.Parsed == nil || rule.TargetImage.Parsed == nil { return nil } - // Cleanup temporary directories - defer func() { - if err := os.RemoveAll(rule.SourcePath); err != nil { - logrus.WithError(err).Warnf("cleanup source image directory %s", rule.SourcePath) - } - if err := os.RemoveAll(rule.NydusdConfig.MountPath); err != nil { - logrus.WithError(err).Warnf("cleanup nydus image directory %s", rule.NydusdConfig.MountPath) - } - if err := os.RemoveAll(rule.NydusdConfig.BlobCacheDir); err != nil { - logrus.WithError(err).Warnf("cleanup nydus blob cache directory %s", rule.NydusdConfig.BlobCacheDir) - } - }() - - image, err := rule.mountSourceImage() + umountSource, err := rule.mountImage(rule.SourceImage, "source") if err != nil { return err } - defer image.Umount() + defer umountSource() - nydusd, err := rule.mountNydusImage() + umountTarget, err := rule.mountImage(rule.TargetImage, "target") if err != nil { return err } - defer nydusd.Umount(false) + defer umountTarget() - return rule.verify() + return rule.verify( + filepath.Join(rule.WorkDir, "source/mnt"), + filepath.Join(rule.WorkDir, "target/mnt"), + ) } diff --git a/contrib/nydusify/pkg/checker/rule/manifest.go b/contrib/nydusify/pkg/checker/rule/manifest.go index cb3c96f2763..6dd7bf58f8f 100644 --- a/contrib/nydusify/pkg/checker/rule/manifest.go +++ b/contrib/nydusify/pkg/checker/rule/manifest.go @@ -6,65 +6,69 @@ package rule import ( "encoding/json" + "fmt" "reflect" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" ) -// ManifestRule validates manifest format of Nydus image +// ManifestRule validates manifest format of nydus image type ManifestRule struct { - SourceParsed *parser.Parsed - TargetParsed *parser.Parsed - MultiPlatform bool - BackendType string - ExpectedArch string + SourceParsed *parser.Parsed + TargetParsed *parser.Parsed } func (rule *ManifestRule) Name() string { - return "Manifest" + return "manifest" } -func (rule *ManifestRule) Validate() error { - logrus.Infof("Checking Nydus manifest") +func (rule *ManifestRule) validateConfig(sourceImage, targetImage *parser.Image) error { + //nolint:staticcheck + // ignore static check SA1019 here. We have to assign deprecated field. + // + // Skip ArgsEscaped's Check + // + // This field is present only for legacy compatibility with Docker and + // should not be used by new image builders. Nydusify (1.6 and above) + // ignores it, which is an expected behavior. + // Also ignore it in check. + // + // Addition: [ArgsEscaped in spec](https://github.com/opencontainers/image-spec/pull/892) + sourceImage.Config.Config.ArgsEscaped = targetImage.Config.Config.ArgsEscaped - // Ensure the target image represents a manifest list, - // and it should consist of OCI and Nydus manifest - if rule.MultiPlatform { - if rule.TargetParsed.Index == nil { - return errors.New("not found image manifest list") - } - foundNydusDesc := false - foundOCIDesc := false - for _, desc := range rule.TargetParsed.Index.Manifests { - if desc.Platform == nil { - continue - } - if desc.Platform.Architecture == rule.ExpectedArch && desc.Platform.OS == "linux" { - if utils.IsNydusPlatform(desc.Platform) { - foundNydusDesc = true - } else { - foundOCIDesc = true - } - } - } - if !foundNydusDesc { - return errors.Errorf("not found nydus image of specified platform linux/%s", rule.ExpectedArch) - } - if !foundOCIDesc { - return errors.Errorf("not found OCI image of specified platform linux/%s", rule.ExpectedArch) - } + sourceConfig, err := json.Marshal(sourceImage.Config.Config) + if err != nil { + return errors.New("marshal source image config") + } + targetConfig, err := json.Marshal(targetImage.Config.Config) + if err != nil { + return errors.New("marshal target image config") } + if !reflect.DeepEqual(sourceConfig, targetConfig) { + return errors.New("source image config should be equal with target image config") + } + + return nil +} - // Check manifest of Nydus - if rule.TargetParsed.NydusImage == nil { - return errors.New("invalid nydus image manifest") +func (rule *ManifestRule) validateOCI(image *parser.Image) error { + // Check config diff IDs + layers := image.Manifest.Layers + if len(image.Config.RootFS.DiffIDs) != len(layers) { + return fmt.Errorf("invalid diff ids in image config: %d (diff ids) != %d (layers)", len(image.Config.RootFS.DiffIDs), len(layers)) } - layers := rule.TargetParsed.NydusImage.Manifest.Layers + return nil +} + +func (rule *ManifestRule) validateNydus(image *parser.Image) error { + // Check bootstrap and blob layers + layers := image.Manifest.Layers for i, layer := range layers { if i == len(layers)-1 { if layer.Annotations[utils.LayerAnnotationNydusBootstrap] != "true" { @@ -78,32 +82,49 @@ func (rule *ManifestRule) Validate() error { } } - // Check Nydus image config with OCI image - if rule.SourceParsed.OCIImage != nil { - - //nolint:staticcheck - // ignore static check SA1019 here. We have to assign deprecated field. - // - // Skip ArgsEscaped's Check - // - // This field is present only for legacy compatibility with Docker and - // should not be used by new image builders. Nydusify (1.6 and above) - // ignores it, which is an expected behavior. - // Also ignore it in check. - // - // Addition: [ArgsEscaped in spec](https://github.com/opencontainers/image-spec/pull/892) - rule.TargetParsed.NydusImage.Config.Config.ArgsEscaped = rule.SourceParsed.OCIImage.Config.Config.ArgsEscaped - - ociConfig, err := json.Marshal(rule.SourceParsed.OCIImage.Config.Config) - if err != nil { - return errors.New("marshal oci image config") + // Check config diff IDs + if len(image.Config.RootFS.DiffIDs) != len(layers) { + return fmt.Errorf("invalid diff ids in image config: %d (diff ids) != %d (layers)", len(image.Config.RootFS.DiffIDs), len(layers)) + } + + return nil +} + +func (rule *ManifestRule) validate(parsed *parser.Parsed) error { + if parsed == nil { + return nil + } + + logrus.WithField("type", tool.CheckImageType(parsed)).WithField("image", parsed.Remote.Ref).Infof("checking manifest") + if parsed.OCIImage != nil { + return errors.Wrap(rule.validateOCI(parsed.OCIImage), "invalid OCI image manifest") + } else if parsed.NydusImage != nil { + return errors.Wrap(rule.validateNydus(parsed.NydusImage), "invalid nydus image manifest") + } + + return errors.New("not found valid image") +} + +func (rule *ManifestRule) Validate() error { + if err := rule.validate(rule.SourceParsed); err != nil { + return errors.Wrap(err, "source image: invalid manifest") + } + + if err := rule.validate(rule.TargetParsed); err != nil { + return errors.Wrap(err, "target image: invalid manifest") + } + + if rule.SourceParsed != nil && rule.TargetParsed != nil { + sourceImage := rule.SourceParsed.OCIImage + if sourceImage == nil { + sourceImage = rule.SourceParsed.NydusImage } - nydusConfig, err := json.Marshal(rule.TargetParsed.NydusImage.Config.Config) - if err != nil { - return errors.New("marshal nydus image config") + targetImage := rule.TargetParsed.OCIImage + if targetImage == nil { + targetImage = rule.TargetParsed.NydusImage } - if !reflect.DeepEqual(ociConfig, nydusConfig) { - return errors.New("nydus image config should be equal with oci image config") + if err := rule.validateConfig(sourceImage, targetImage); err != nil { + return fmt.Errorf("validate image config: %v", err) } } diff --git a/contrib/nydusify/pkg/checker/rule/manifest_test.go b/contrib/nydusify/pkg/checker/rule/manifest_test.go index a6f3e3c3a3a..e9eff2234a5 100644 --- a/contrib/nydusify/pkg/checker/rule/manifest_test.go +++ b/contrib/nydusify/pkg/checker/rule/manifest_test.go @@ -56,13 +56,11 @@ func TestManifestRuleValidate_MultiPlatform(t *testing.T) { } rule := ManifestRule{ - MultiPlatform: true, - ExpectedArch: "amd64", - SourceParsed: source, - TargetParsed: target, + SourceParsed: source, + TargetParsed: target, } require.Error(t, rule.Validate()) - require.Contains(t, rule.Validate().Error(), "not found image manifest list") + require.Contains(t, rule.Validate().Error(), "not found manifest list") rule.TargetParsed.Index = &ocispec.Index{} require.Error(t, rule.Validate()) diff --git a/contrib/nydusify/pkg/checker/tool/builder.go b/contrib/nydusify/pkg/checker/tool/builder.go index 50dca87d1cc..0df02e11c43 100644 --- a/contrib/nydusify/pkg/checker/tool/builder.go +++ b/contrib/nydusify/pkg/checker/tool/builder.go @@ -29,7 +29,7 @@ func NewBuilder(binaryPath string) *Builder { } } -// Check calls `nydus-image check` to parse Nydus bootstrap +// Check calls `nydus-image check` to parse nydus bootstrap // and output debug information to specified JSON file. func (builder *Builder) Check(option BuilderOption) error { args := []string{ diff --git a/contrib/nydusify/pkg/checker/tool/image.go b/contrib/nydusify/pkg/checker/tool/image.go index af9d7099cf6..9ab14c6f1fd 100644 --- a/contrib/nydusify/pkg/checker/tool/image.go +++ b/contrib/nydusify/pkg/checker/tool/image.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/containerd/containerd/mount" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -45,11 +46,19 @@ func mkMounts(dirs []string) []mount.Mount { } } +func CheckImageType(parsed *parser.Parsed) string { + if parsed.NydusImage != nil { + return "nydus" + } else if parsed.OCIImage != nil { + return "oci" + } + return "unknown" +} + type Image struct { - Layers []ocispec.Descriptor - Source string - SourcePath string - Rootfs string + Layers []ocispec.Descriptor + LayerBaseDir string + Rootfs string } // Mount mounts rootfs of OCI image. @@ -62,7 +71,7 @@ func (image *Image) Mount() error { count := len(image.Layers) for idx := range image.Layers { layerName := fmt.Sprintf("layer-%d", count-idx-1) - layerDir := filepath.Join(image.SourcePath, layerName) + layerDir := filepath.Join(image.LayerBaseDir, layerName) dirs = append(dirs, strings.ReplaceAll(layerDir, ":", "\\:")) } diff --git a/contrib/nydusify/pkg/parser/parser.go b/contrib/nydusify/pkg/parser/parser.go index 64834242d60..bedc0466c40 100644 --- a/contrib/nydusify/pkg/parser/parser.go +++ b/contrib/nydusify/pkg/parser/parser.go @@ -42,7 +42,8 @@ type Image struct { // Parsed presents OCI and Nydus image manifest. // Nydus image conversion only works on top of an existed oci image whose platform is linux/amd64 type Parsed struct { - Index *ocispec.Index + Remote *remote.Remote + Index *ocispec.Index // The base image from which to generate nydus image. OCIImage *Image NydusImage *Image @@ -183,9 +184,9 @@ func (parser *Parser) matchImagePlatform(desc *ocispec.Descriptor) bool { // Parse parses Nydus image reference into Parsed object. func (parser *Parser) Parse(ctx context.Context) (*Parsed, error) { - logrus.Infof("Parsing image %s", parser.Remote.Ref) - - parsed := Parsed{} + parsed := Parsed{ + Remote: parser.Remote, + } imageDesc, err := parser.Remote.Resolve(ctx) if err != nil { diff --git a/contrib/nydusify/pkg/utils/utils.go b/contrib/nydusify/pkg/utils/utils.go index 7be572a8034..d376ae96381 100644 --- a/contrib/nydusify/pkg/utils/utils.go +++ b/contrib/nydusify/pkg/utils/utils.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "os" + "path/filepath" "runtime" "strings" "syscall" @@ -167,6 +168,45 @@ func UnpackFile(reader io.Reader, source, target string) error { return nil } +func UnpackFromTar(reader io.Reader, targetDir string) error { + if err := os.MkdirAll(targetDir, 0755); err != nil { + return err + } + + tr := tar.NewReader(reader) + for { + header, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + filePath := filepath.Join(targetDir, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(filePath, header.FileInfo().Mode()); err != nil { + return err + } + case tar.TypeReg: + f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, header.FileInfo().Mode()) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, tr); err != nil { + return err + } + default: + } + } + + return nil +} + func IsEmptyString(str string) bool { return strings.TrimSpace(str) == "" }