Skip to content

Commit

Permalink
nydusify: refactor check subcommand
Browse files Browse the repository at this point in the history 
- allow either the source or target to be an OCI or nydus image;
- improve output directory structure and log format;

Signed-off-by: Yan Song <[email protected]>
  • Loading branch information
@imeoer
imeoer committed Dec 11, 2024
1 parent 375f55f commit d294237
Show file tree
Hide file tree
Showing 11 changed files with 536 additions and 381 deletions.
55 changes: 42 additions & 13 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -559,19 +559,39 @@ func main() {
},

&cli.StringFlag{
Name: "backend-type",
Name: "source-backend-type",
Value: "",
Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3'",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "backend-config",
Name: "source-backend-config",
Value: "",
Usage: "Json string for storage backend configuration",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "backend-config-file",
Name: "source-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
EnvVars: []string{"BACKEND_CONFIG_FILE"},
},

&cli.StringFlag{
Name: "target-backend-type",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "target-backend-config",
Value: "",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "target-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
Expand Down Expand Up @@ -612,7 +632,12 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, "", false)
sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
if err != nil {
return err
}

targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false)
if err != nil {
return err
}
Expand All @@ -623,16 +648,20 @@ func main() {
}

checker, err := checker.New(checker.Opt{
WorkDir: c.String("work-dir"),
Source: c.String("source"),
Target: c.String("target"),
WorkDir: c.String("work-dir"),

Source: c.String("source"),
Target: c.String("target"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),
SourceBackendType: sourceBackendType,
SourceBackendConfig: sourceBackendConfig,
TargetBackendType: targetBackendType,
TargetBackendConfig: targetBackendConfig,

MultiPlatform: c.Bool("multi-platform"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),
NydusImagePath: c.String("nydus-image"),
NydusdPath: c.String("nydusd"),
BackendType: backendType,
BackendConfig: backendConfig,
ExpectedArch: arch,
})
if err != nil {
Expand Down
131 changes: 55 additions & 76 deletions contrib/nydusify/pkg/checker/checker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,44 @@ import (
"github.com/sirupsen/logrus"

"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
)

// Opt defines Checker options.
// Note: target is the Nydus image reference.
// Note: target is the nydus image reference.
type Opt struct {
WorkDir string
Source string
Target string
SourceInsecure bool
TargetInsecure bool
WorkDir string

Source string
Target string
SourceInsecure bool
TargetInsecure bool
SourceBackendType string
SourceBackendConfig string
TargetBackendType string
TargetBackendConfig string

MultiPlatform bool
NydusImagePath string
NydusdPath string
BackendType string
BackendConfig string
ExpectedArch string
}

// Checker validates Nydus image manifest, bootstrap and mounts filesystem
// by Nydusd to compare file metadata and data with OCI image.
// Checker validates nydus image manifest, bootstrap and mounts filesystem
// by nydusd to compare file metadata and data between OCI / nydus image.
type Checker struct {
Opt
sourceParser *parser.Parser
targetParser *parser.Parser
}

// New creates Checker instance, target is the Nydus image reference.
// New creates Checker instance, target is the nydus image reference.
func New(opt Opt) (*Checker, error) {
// TODO: support source and target resolver
targetRemote, err := provider.DefaultRemote(opt.Target, opt.TargetInsecure)
if err != nil {
return nil, errors.Wrap(err, "Init target image parser")
return nil, errors.Wrap(err, "init target image parser")
}
targetParser, err := parser.New(targetRemote, opt.ExpectedArch)
if err != nil {
Expand All @@ -63,7 +64,7 @@ func New(opt Opt) (*Checker, error) {
return nil, errors.Wrap(err, "Init source image parser")
}
sourceParser, err = parser.New(sourceRemote, opt.ExpectedArch)
if sourceParser == nil {
if err != nil {
return nil, errors.Wrap(err, "failed to create parser")
}
}
Expand All @@ -77,7 +78,7 @@ func New(opt Opt) (*Checker, error) {
return checker, nil
}

// Check checks Nydus image, and outputs image information to work
// Check checks nydus image, and outputs image information to work
// directory, the check workflow is composed of various rules.
func (checker *Checker) Check(ctx context.Context) error {
if err := checker.check(ctx); err != nil {
Expand All @@ -93,12 +94,13 @@ func (checker *Checker) Check(ctx context.Context) error {
return nil
}

// Check checks Nydus image, and outputs image information to work
// Check checks nydus image, and outputs image information to work
// directory, the check workflow is composed of various rules.
func (checker *Checker) check(ctx context.Context) error {
logrus.WithField("image", checker.targetParser.Remote.Ref).Infof("parsing image")
targetParsed, err := checker.targetParser.Parse(ctx)
if err != nil {
return errors.Wrap(err, "parse Nydus image")
return errors.Wrap(err, "parse nydus image")
}

var sourceParsed *parser.Parsed
Expand All @@ -107,89 +109,66 @@ func (checker *Checker) check(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "parse source image")
}
} else {
sourceParsed = targetParsed
}

if err := os.RemoveAll(checker.WorkDir); err != nil {
return errors.Wrap(err, "clean up work directory")
}

if err := os.MkdirAll(filepath.Join(checker.WorkDir, "fs"), 0755); err != nil {
return errors.Wrap(err, "create work directory")
}

if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil {
return errors.Wrap(err, "output image information")
}

mode := "direct"
digestValidate := false
if targetParsed.NydusImage != nil {
nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest)
if nydusManifest != nil {
v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5)
if v == utils.V5 {
// Digest validate is not currently supported for v6,
// but v5 supports it. In order to make the check more sufficient,
// this validate needs to be turned on for v5.
digestValidate = true
}
if sourceParsed != nil {
if err := checker.Output(ctx, sourceParsed, filepath.Join(checker.WorkDir, "source")); err != nil {
return errors.Wrapf(err, "output image information: %s", sourceParsed.Remote.Ref)
}
}

var sourceRemote *remote.Remote
if checker.sourceParser != nil {
sourceRemote = checker.sourceParser.Remote
if targetParsed != nil {
if err := checker.Output(ctx, targetParsed, filepath.Join(checker.WorkDir, "target")); err != nil {
return errors.Wrapf(err, "output image information: %s", targetParsed.Remote.Ref)
}
}

rules := []rule.Rule{
&rule.ManifestRule{
SourceParsed: sourceParsed,
TargetParsed: targetParsed,
MultiPlatform: checker.MultiPlatform,
BackendType: checker.BackendType,
ExpectedArch: checker.ExpectedArch,
SourceParsed: sourceParsed,
TargetParsed: targetParsed,
},
&rule.BootstrapRule{
Parsed: targetParsed,
NydusImagePath: checker.NydusImagePath,
BackendType: checker.BackendType,
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
DebugOutputPath: filepath.Join(checker.WorkDir, "nydus_bootstrap_debug.json"),
WorkDir: checker.WorkDir,
NydusImagePath: checker.NydusImagePath,

SourceParsed: sourceParsed,
TargetParsed: targetParsed,
SourceBackendType: checker.SourceBackendType,
SourceBackendConfig: checker.SourceBackendConfig,
TargetBackendType: checker.TargetBackendType,
TargetBackendConfig: checker.TargetBackendConfig,
},
&rule.FilesystemRule{
Source: checker.Source,
SourceMountPath: filepath.Join(checker.WorkDir, "fs/source_mounted"),
SourceParsed: sourceParsed,
SourcePath: filepath.Join(checker.WorkDir, "fs/source"),
SourceRemote: sourceRemote,
Target: checker.Target,
TargetInsecure: checker.TargetInsecure,
PlainHTTP: checker.targetParser.Remote.IsWithHTTP(),
NydusdConfig: tool.NydusdConfig{
EnablePrefetch: true,
NydusdPath: checker.NydusdPath,
BackendType: checker.BackendType,
BackendConfig: checker.BackendConfig,
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
ConfigPath: filepath.Join(checker.WorkDir, "fs/nydusd_config.json"),
BlobCacheDir: filepath.Join(checker.WorkDir, "fs/nydus_blobs"),
MountPath: filepath.Join(checker.WorkDir, "fs/nydus_mounted"),
APISockPath: filepath.Join(checker.WorkDir, "fs/nydus_api.sock"),
Mode: mode,
DigestValidate: digestValidate,
WorkDir: checker.WorkDir,
NydusdPath: checker.NydusdPath,

SourceImage: &rule.Image{
Parsed: sourceParsed,
Insecure: checker.SourceInsecure,
},
TargetImage: &rule.Image{
Parsed: targetParsed,
Insecure: checker.TargetInsecure,
},
SourceBackendType: checker.SourceBackendType,
SourceBackendConfig: checker.SourceBackendConfig,
TargetBackendType: checker.TargetBackendType,
TargetBackendConfig: checker.TargetBackendConfig,
},
}

for _, rule := range rules {
if err := rule.Validate(); err != nil {
return errors.Wrapf(err, "validate rule %s", rule.Name())
return errors.Wrapf(err, "validate %s failed", rule.Name())
}
}

logrus.Infof("Verified Nydus image %s", checker.targetParser.Remote.Ref)
logrus.Info("verified image")

return nil
}
Loading

0 comments on commit d294237

Please sign in to comment.