Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nydusify: refactor check subcommand #1652

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

nydusify: refactor check subcommand #1652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 42 additions & 13 deletions contrib/nydusify/cmd/nydusify.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,19 +559,39 @@ func main() {
},

&cli.StringFlag{
Name: "backend-type",
Name: "source-backend-type",
Value: "",
Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3'",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "backend-config",
Name: "source-backend-config",
Value: "",
Usage: "Json string for storage backend configuration",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "backend-config-file",
Name: "source-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
EnvVars: []string{"BACKEND_CONFIG_FILE"},
},

&cli.StringFlag{
Name: "target-backend-type",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "target-backend-config",
Value: "",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "target-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
Expand Down Expand Up @@ -612,7 +632,12 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, "", false)
sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
if err != nil {
return err
}

targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false)
if err != nil {
return err
}
Expand All @@ -623,16 +648,20 @@ func main() {
}

checker, err := checker.New(checker.Opt{
WorkDir: c.String("work-dir"),
Source: c.String("source"),
Target: c.String("target"),
WorkDir: c.String("work-dir"),

Source: c.String("source"),
Target: c.String("target"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),
SourceBackendType: sourceBackendType,
SourceBackendConfig: sourceBackendConfig,
TargetBackendType: targetBackendType,
TargetBackendConfig: targetBackendConfig,

MultiPlatform: c.Bool("multi-platform"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),
NydusImagePath: c.String("nydus-image"),
NydusdPath: c.String("nydusd"),
BackendType: backendType,
BackendConfig: backendConfig,
ExpectedArch: arch,
})
if err != nil {
Expand Down
131 changes: 55 additions & 76 deletions contrib/nydusify/pkg/checker/checker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,44 @@ import (
"github.com/sirupsen/logrus"

"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
)

// Opt defines Checker options.
// Note: target is the Nydus image reference.
// Note: target is the nydus image reference.
type Opt struct {
WorkDir string
Source string
Target string
SourceInsecure bool
TargetInsecure bool
WorkDir string

Source string
Target string
SourceInsecure bool
TargetInsecure bool
SourceBackendType string
SourceBackendConfig string
TargetBackendType string
TargetBackendConfig string

MultiPlatform bool
NydusImagePath string
NydusdPath string
BackendType string
BackendConfig string
ExpectedArch string
}

// Checker validates Nydus image manifest, bootstrap and mounts filesystem
// by Nydusd to compare file metadata and data with OCI image.
// Checker validates nydus image manifest, bootstrap and mounts filesystem
// by nydusd to compare file metadata and data between OCI / nydus image.
type Checker struct {
Opt
sourceParser *parser.Parser
targetParser *parser.Parser
}

// New creates Checker instance, target is the Nydus image reference.
// New creates Checker instance, target is the nydus image reference.
func New(opt Opt) (*Checker, error) {
// TODO: support source and target resolver
targetRemote, err := provider.DefaultRemote(opt.Target, opt.TargetInsecure)
if err != nil {
return nil, errors.Wrap(err, "Init target image parser")
return nil, errors.Wrap(err, "init target image parser")
}
targetParser, err := parser.New(targetRemote, opt.ExpectedArch)
if err != nil {
Expand All @@ -63,7 +64,7 @@ func New(opt Opt) (*Checker, error) {
return nil, errors.Wrap(err, "Init source image parser")
}
sourceParser, err = parser.New(sourceRemote, opt.ExpectedArch)
if sourceParser == nil {
if err != nil {
return nil, errors.Wrap(err, "failed to create parser")
}
}
Expand All @@ -77,7 +78,7 @@ func New(opt Opt) (*Checker, error) {
return checker, nil
}

// Check checks Nydus image, and outputs image information to work
// Check checks nydus image, and outputs image information to work
// directory, the check workflow is composed of various rules.
func (checker *Checker) Check(ctx context.Context) error {
if err := checker.check(ctx); err != nil {
Expand All @@ -93,12 +94,13 @@ func (checker *Checker) Check(ctx context.Context) error {
return nil
}

// Check checks Nydus image, and outputs image information to work
// Check checks nydus image, and outputs image information to work
// directory, the check workflow is composed of various rules.
func (checker *Checker) check(ctx context.Context) error {
logrus.WithField("image", checker.targetParser.Remote.Ref).Infof("parsing image")
targetParsed, err := checker.targetParser.Parse(ctx)
if err != nil {
return errors.Wrap(err, "parse Nydus image")
return errors.Wrap(err, "parse nydus image")
}

var sourceParsed *parser.Parsed
Expand All @@ -107,89 +109,66 @@ func (checker *Checker) check(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "parse source image")
}
} else {
sourceParsed = targetParsed
}

if err := os.RemoveAll(checker.WorkDir); err != nil {
return errors.Wrap(err, "clean up work directory")
}

if err := os.MkdirAll(filepath.Join(checker.WorkDir, "fs"), 0755); err != nil {
return errors.Wrap(err, "create work directory")
}

if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil {
return errors.Wrap(err, "output image information")
}

mode := "direct"
digestValidate := false
if targetParsed.NydusImage != nil {
nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest)
if nydusManifest != nil {
v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5)
if v == utils.V5 {
// Digest validate is not currently supported for v6,
// but v5 supports it. In order to make the check more sufficient,
// this validate needs to be turned on for v5.
digestValidate = true
}
if sourceParsed != nil {
if err := checker.Output(ctx, sourceParsed, filepath.Join(checker.WorkDir, "source")); err != nil {
return errors.Wrapf(err, "output image information: %s", sourceParsed.Remote.Ref)
}
}

var sourceRemote *remote.Remote
if checker.sourceParser != nil {
sourceRemote = checker.sourceParser.Remote
if targetParsed != nil {
if err := checker.Output(ctx, targetParsed, filepath.Join(checker.WorkDir, "target")); err != nil {
return errors.Wrapf(err, "output image information: %s", targetParsed.Remote.Ref)
}
}

rules := []rule.Rule{
&rule.ManifestRule{
SourceParsed: sourceParsed,
TargetParsed: targetParsed,
MultiPlatform: checker.MultiPlatform,
BackendType: checker.BackendType,
ExpectedArch: checker.ExpectedArch,
SourceParsed: sourceParsed,
TargetParsed: targetParsed,
},
&rule.BootstrapRule{
Parsed: targetParsed,
NydusImagePath: checker.NydusImagePath,
BackendType: checker.BackendType,
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
DebugOutputPath: filepath.Join(checker.WorkDir, "nydus_bootstrap_debug.json"),
WorkDir: checker.WorkDir,
NydusImagePath: checker.NydusImagePath,

SourceParsed: sourceParsed,
TargetParsed: targetParsed,
SourceBackendType: checker.SourceBackendType,
SourceBackendConfig: checker.SourceBackendConfig,
TargetBackendType: checker.TargetBackendType,
TargetBackendConfig: checker.TargetBackendConfig,
},
&rule.FilesystemRule{
Source: checker.Source,
SourceMountPath: filepath.Join(checker.WorkDir, "fs/source_mounted"),
SourceParsed: sourceParsed,
SourcePath: filepath.Join(checker.WorkDir, "fs/source"),
SourceRemote: sourceRemote,
Target: checker.Target,
TargetInsecure: checker.TargetInsecure,
PlainHTTP: checker.targetParser.Remote.IsWithHTTP(),
NydusdConfig: tool.NydusdConfig{
EnablePrefetch: true,
NydusdPath: checker.NydusdPath,
BackendType: checker.BackendType,
BackendConfig: checker.BackendConfig,
BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
ConfigPath: filepath.Join(checker.WorkDir, "fs/nydusd_config.json"),
BlobCacheDir: filepath.Join(checker.WorkDir, "fs/nydus_blobs"),
MountPath: filepath.Join(checker.WorkDir, "fs/nydus_mounted"),
APISockPath: filepath.Join(checker.WorkDir, "fs/nydus_api.sock"),
Mode: mode,
DigestValidate: digestValidate,
WorkDir: checker.WorkDir,
NydusdPath: checker.NydusdPath,

SourceImage: &rule.Image{
Parsed: sourceParsed,
Insecure: checker.SourceInsecure,
},
TargetImage: &rule.Image{
Parsed: targetParsed,
Insecure: checker.TargetInsecure,
},
SourceBackendType: checker.SourceBackendType,
SourceBackendConfig: checker.SourceBackendConfig,
TargetBackendType: checker.TargetBackendType,
TargetBackendConfig: checker.TargetBackendConfig,
},
}

for _, rule := range rules {
if err := rule.Validate(); err != nil {
return errors.Wrapf(err, "validate rule %s", rule.Name())
return errors.Wrapf(err, "validate %s failed", rule.Name())
}
}

logrus.Infof("Verified Nydus image %s", checker.targetParser.Remote.Ref)
logrus.Info("verified image")

return nil
}
Loading
Loading