Skip to content

Commit

Permalink
Copy full project (#5033)
Browse files Browse the repository at this point in the history
  • Loading branch information
modulo11 authored Sep 6, 2024
1 parent 72ff2d4 commit 34f1057
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 18 deletions.
4 changes: 2 additions & 2 deletions cmd/cnbBuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image
}

if pathType != buildpacks.PathEnumArchive {
err = cnbutils.CopyProject(source, target, include, exclude, utils)
err = cnbutils.CopyProject(source, target, include, exclude, utils, false)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target)
Expand Down Expand Up @@ -619,7 +619,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image

if len(config.PreserveFiles) > 0 {
if pathType != buildpacks.PathEnumArchive {
err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils)
err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils, true)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles)
Expand Down
81 changes: 73 additions & 8 deletions pkg/cnbutils/copy_project.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cnbutils

import (
"io/fs"
"os"
"path"
"path/filepath"
Expand All @@ -11,23 +12,82 @@ import (
ignore "github.com/sabhiram/go-gitignore"
)

func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils) error {
sourceFiles, _ := utils.Glob(path.Join(source, "**"))
func shouldBeFiltered(path string, knownSymlinks []string) bool {
for _, symlink := range knownSymlinks {
if strings.HasPrefix(path, symlink) {
return true
}
}
return false
}

func filterSymlinks(sourceFiles []string, utils BuildUtils) ([]string, error) {
filteredFiles := []string{}
knownSymlinks := []string{}

for _, sourceFile := range sourceFiles {
if shouldBeFiltered(sourceFile, knownSymlinks) {
continue
}

isSymlink, err := symlinkExists(sourceFile, utils)
if err != nil {
return nil, err
}

if isSymlink {
log.Entry().Debugf("Ignoring any path below %q", sourceFile)
knownSymlinks = append(knownSymlinks, sourceFile)
}
filteredFiles = append(filteredFiles, sourceFile)
}
return filteredFiles, nil
}

func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils, follow bool) error {
sourceFiles, err := utils.Glob(path.Join(source, "**"))
if err != nil {
return err
}

if !follow {
sourceFiles, err = filterSymlinks(sourceFiles, utils)
if err != nil {
return err
}
}

for _, sourceFile := range sourceFiles {
relPath, err := filepath.Rel(source, sourceFile)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Calculating relative path for '%s' failed", sourceFile)
}

if !isIgnored(relPath, include, exclude) {
target := path.Join(target, strings.ReplaceAll(sourceFile, source, ""))
dir, err := utils.DirExists(sourceFile)

isSymlink, err := symlinkExists(sourceFile, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Checking file info '%s' failed", target)
return err
}

if dir {
isDir, err := utils.DirExists(sourceFile)
if err != nil {
return err
}

if isSymlink {
linkTarget, err := utils.Readlink(sourceFile)
if err != nil {
return err
}
log.Entry().Debugf("Creating symlink from %q to %q", target, linkTarget)
err = utils.Symlink(linkTarget, target)
if err != nil {
return err
}
} else if isDir {
err = utils.MkdirAll(target, os.ModePerm)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
Expand All @@ -41,12 +101,16 @@ func CopyProject(source, target string, include, exclude *ignore.GitIgnore, util
return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target)
}
}

}
}
return nil
}

func symlinkExists(path string, utils BuildUtils) (bool, error) {
lstat, err := utils.Lstat(path)
return lstat.Mode().Type() == fs.ModeSymlink, err
}

func copyFile(source, target string, utils BuildUtils) error {
targetDir := filepath.Dir(target)

Expand All @@ -56,13 +120,14 @@ func copyFile(source, target string, utils BuildUtils) error {
}

if !exists {
log.Entry().Debugf("Creating directory %s", targetDir)
log.Entry().Debugf("Creating directory '%s'", targetDir)
err = utils.MkdirAll(targetDir, os.ModePerm)
if err != nil {
return err
}
}

log.Entry().Debugf("Copying '%s' to '%s'", source, target)
_, err = utils.Copy(source, target)
return err
}
Expand Down
28 changes: 22 additions & 6 deletions pkg/cnbutils/copy_project_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,43 @@ import (
)

func TestCopyProject(t *testing.T) {
t.Run("copies file according to doublestart globs", func(t *testing.T) {
t.Run("copy project with following symlinks", func(t *testing.T) {
mockUtils := &cnbutils.MockUtils{
FilesMock: &mock.FilesMock{},
}
mockUtils.AddFile("workdir/src/test.yaml", []byte(""))
mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte(""))
mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte(""))
err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils)

mockUtils.AddDir("workdir/apps")
mockUtils.AddFile("workdir/apps/foo.yaml", []byte(""))
mockUtils.Symlink("workdir/apps", "/workdir/src/apps")

err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**"}...), nil, mockUtils, true)
assert.NoError(t, err)
assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/apps", "/dest/apps"))
})

t.Run("copies file according to simple globs", func(t *testing.T) {
t.Run("copy project without following symlinks", func(t *testing.T) {
mockUtils := &cnbutils.MockUtils{
FilesMock: &mock.FilesMock{},
}
mockUtils.AddFile("src/test.yaml", []byte(""))
err := cnbutils.CopyProject("src", "/dest", ignore.CompileIgnoreLines([]string{"*.yaml"}...), nil, mockUtils)
mockUtils.AddFile("workdir/src/test.yaml", []byte(""))
mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte(""))
mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte(""))

mockUtils.AddDir("workdir/apps")
mockUtils.AddFile("workdir/apps/foo.yaml", []byte(""))
mockUtils.Symlink("workdir/apps", "/workdir/src/apps")

err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils, false)
assert.NoError(t, err)
assert.True(t, mockUtils.HasCopiedFile("src/test.yaml", "/dest/test.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
assert.True(t, mockUtils.HasCreatedSymlink("workdir/apps", "/workdir/src/apps"))
})
}
17 changes: 15 additions & 2 deletions pkg/mock/fileUtils.go
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,10 @@ func (f *FilesMock) Stat(path string) (os.FileInfo, error) {
}, nil
}

func (f *FilesMock) Lstat(path string) (os.FileInfo, error) {
return f.Stat(path)
}

// Chmod changes the file mode for the entry at the given path
func (f *FilesMock) Chmod(path string, mode os.FileMode) error {
props, exists := f.files[f.toAbsPath(path)]
Expand Down Expand Up @@ -540,8 +544,9 @@ func (f *FilesMock) Symlink(oldname, newname string) error {
f.init()

f.files[newname] = &fileProperties{
isLink: true,
target: oldname,
isLink: true,
target: oldname,
content: &[]byte{},
}

return nil
Expand Down Expand Up @@ -700,3 +705,11 @@ func (f *FilesMockRelativeGlob) Glob(pattern string) ([]string, error) {
sort.Strings(matches)
return matches, nil
}

func (f *FilesMock) Readlink(name string) (string, error) {
properties, ok := f.files[name]
if ok && properties.isLink {
return properties.target, nil
}
return "", fmt.Errorf("could not retrieve target for %s", name)
}
13 changes: 13 additions & 0 deletions pkg/piperutils/fileUtils.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ type FileUtils interface {
CurrentTime(format string) string
Open(name string) (io.ReadWriteCloser, error)
Create(name string) (io.ReadWriteCloser, error)
Readlink(name string) (string, error)
Stat(path string) (os.FileInfo, error)
Lstat(path string) (os.FileInfo, error)
}

// Files ...
Expand Down Expand Up @@ -513,3 +516,13 @@ func (f Files) Open(name string) (io.ReadWriteCloser, error) {
func (f Files) Create(name string) (io.ReadWriteCloser, error) {
return os.Create(name)
}

// Readlink wraps os.Readlink
func (f Files) Readlink(name string) (string, error) {
return os.Readlink(name)
}

// Readlink wraps os.Readlink
func (f Files) Lstat(path string) (os.FileInfo, error) {
return os.Lstat(path)
}

0 comments on commit 34f1057

Please sign in to comment.