implement option for blacklisting files

.gitlab-ci.yml

@@ -1,7 +1,7 @@
 variables:
   GITHUB_USER: rainu
   GITHUB_REPO: backup2glacier
-  RELEASE_TAG: v0.1.0
+  RELEASE_TAG: v0.1.1
 
 stages:
 - build

README.md

@@ -53,6 +53,9 @@ go build
 
 ## Release History
 
+* 0.1.1
+    * CLI option for blacklisting files
+    * Correct default value (1) of option for upload part size
 * 0.1.0
     * Backup will not exit on files which does not exists (dead links)
 * 0.0.2

backup/manager.go

@@ -7,6 +7,7 @@ import (
 	"github.com/pkg/errors"
 	"io"
 	"os"
+	"regexp"
 	"sync"
 	"time"
 )
@@ -24,7 +25,7 @@ type BackupResult struct {
 type BackupCreater interface {
 	io.Closer
 
-	Create(files []string, description, vaultName string) *BackupResult
+	Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult
 }
 
 type BackupGetter interface {
@@ -40,7 +41,7 @@ type BackupDeleter interface {
 type BackupManager interface {
 	io.Closer
 
-	Create(files []string, description, vaultName string) *BackupResult
+	Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult
 	Download(backupId uint, target string) error
 	Delete(backupId uint) error
 }
@@ -89,7 +90,7 @@ func (b *backupManager) Close() error {
 	return b.dbRepository.Close()
 }
 
-func (b *backupManager) Create(files []string, description, vaultName string) *BackupResult {
+func (b *backupManager) Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult {
 	// folder/file -> zip -> encrypt -> glacier
 	srcZip, dstZip := io.Pipe()
 	srcCrypt, dstCrypt := io.Pipe()
@@ -107,7 +108,7 @@ func (b *backupManager) Create(files []string, description, vaultName string) *B
 		defer wg.Done()
 		defer dstZip.Close()
 
-		Zip(files, dstZip, contentChan)
+		Zip(files, blacklist, dstZip, contentChan)
 	}()
 	go func() {
 		for {

backup/manager_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 	"testing"
 )
@@ -27,7 +28,7 @@ func Test_ZipEncryptDecryptUnzip(t *testing.T) {
 	go func() {
 		defer dstZip.Close()
 
-		Zip([]string{"./"}, dstZip, nil)
+		Zip([]string{"./"}, []*regexp.Regexp{}, dstZip, nil)
 	}()
 
 	encodedZipFile, err := ioutil.TempFile("", ".enc")

backup/zip.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 )
 
@@ -19,7 +20,7 @@ type ZipContent struct {
 }
 
 //ZIP the given file/folder and write file information out in given channel
-func Zip(filePaths []string, dst io.Writer, contentChan chan<- *ZipContent) {
+func Zip(filePaths []string, blacklist []*regexp.Regexp, dst io.Writer, contentChan chan<- *ZipContent) {
 	// Create a new zip archive.
 	zipWriter := zip.NewWriter(dst)
 	zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
@@ -36,10 +37,10 @@ func Zip(filePaths []string, dst io.Writer, contentChan chan<- *ZipContent) {
 		}
 
 		if fInfo.IsDir() {
-			addFiles(zipWriter, absFilePath+"/", filepath.Dir(absFilePath+"/")+"/", contentChan)
+			addFiles(zipWriter, absFilePath+"/", filepath.Dir(absFilePath+"/")+"/", blacklist, contentChan)
 		} else {
 			dir, name := filepath.Split(absFilePath)
-			addFile(zipWriter, dir, dir+"/", name, contentChan)
+			addFile(zipWriter, dir, dir+"/", name, blacklist, contentChan)
 		}
 	}
 
@@ -48,7 +49,7 @@ func Zip(filePaths []string, dst io.Writer, contentChan chan<- *ZipContent) {
 	}
 }
 
-func addFiles(w *zip.Writer, basePath, baseInZip string, contentChan chan<- *ZipContent) {
+func addFiles(w *zip.Writer, basePath, baseInZip string, blacklist []*regexp.Regexp, contentChan chan<- *ZipContent) {
 	// Open the Directory
 	files, err := ioutil.ReadDir(basePath)
 	if err != nil {
@@ -60,14 +61,14 @@ func addFiles(w *zip.Writer, basePath, baseInZip string, contentChan chan<- *Zip
 		if fileDesc.IsDir() {
 			// recursion ahead!
 			newBase := basePath + fileDesc.Name() + "/"
-			addFiles(w, newBase, baseInZip+"/"+fileDesc.Name()+"/", contentChan)
+			addFiles(w, newBase, baseInZip+"/"+fileDesc.Name()+"/", blacklist, contentChan)
 		} else {
-			addFile(w, basePath, baseInZip, fileDesc.Name(), contentChan)
+			addFile(w, basePath, baseInZip, fileDesc.Name(), blacklist, contentChan)
 		}
 	}
 }
 
-func addFile(w *zip.Writer, basePath, baseInZip, fileName string, contentChan chan<- *ZipContent) int64 {
+func addFile(w *zip.Writer, basePath, baseInZip, fileName string, blacklist []*regexp.Regexp, contentChan chan<- *ZipContent) int64 {
 	//open for reading
 	filePath := normalizeFilePath(basePath + fileName)
 	zipPath := normalizeZipPath(baseInZip + fileName)
@@ -79,6 +80,11 @@ func addFile(w *zip.Writer, basePath, baseInZip, fileName string, contentChan ch
 	}
 	defer osFile.Close()
 
+	if blacklisted, expr := isBlacklisted(filePath, blacklist); blacklisted {
+		LogInfo(`Ignore file because it is blacklisted: %s -> "%s"`, filePath, expr)
+		return 0
+	}
+
 	LogInfo("Add to zip: %s -> %s", osFile.Name(), zipPath)
 
 	// Add some files to the archive.
@@ -119,6 +125,16 @@ func addFile(w *zip.Writer, basePath, baseInZip, fileName string, contentChan ch
 	return written
 }
 
+func isBlacklisted(path string, blacklist []*regexp.Regexp) (bool, *regexp.Regexp) {
+	for _, curExpr := range blacklist {
+		if curExpr.MatchString(path) {
+			return true, curExpr
+		}
+	}
+
+	return false, nil
+}
+
 func normalizeFilePath(path string) string {
 	return strings.Replace(path, "//", "/", -1)
 }

backup/zip_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 	"sync"
 	"testing"
@@ -35,7 +36,7 @@ func Test_Zip(t *testing.T) {
 		}
 	}()
 
-	Zip([]string{"./"}, tmpFile, contentChan)
+	Zip([]string{"./"}, []*regexp.Regexp{}, tmpFile, contentChan)
 
 	wg.Wait()
 	assert.True(t, containsTestFile)
@@ -54,3 +55,26 @@ func Test_Zip(t *testing.T) {
 	}
 	assert.True(t, containsTestFile)
 }
+
+func TestBlacklist(t *testing.T) {
+	tests := []struct {
+		name           string
+		blacklist      string
+		testCase       string
+		expectedResult bool
+	}{
+		{"suffixes", `.*\.exe`, "test.exe", true },
+		{"suffixes", `.*\.exe`, "test.exe_", true },
+		{"suffixes", `.*\.exe$`, "test.exe_", false },
+		{"suffixes", `.*\.exe`, "full/path/to/test.exe", true },
+		{"suffixes", `.*\.exe`, ".exe", true },
+		{"suffixes", `.*\.exe`, "exe", false },
+		{"folders", `.*/log/.*`, "/path/to/log/test.txt", true },
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result, _ := isBlacklisted(test.testCase, []*regexp.Regexp{regexp.MustCompile(test.blacklist)})
+			assert.Equal(t, result, test.expectedResult)
+		})
+	}
+}

cli/action_create.go

@@ -6,6 +6,7 @@ import (
 	. "backup2glacier/log"
 	"fmt"
 	"golang.org/x/crypto/ssh/terminal"
+	"regexp"
 	"syscall"
 )
 
@@ -30,7 +31,7 @@ func (a *actionCreate) Do(cfg *config.Config) {
 	}
 	defer b.Close()
 
-	result := b.Create(cfg.Create.Files, cfg.Create.AWSArchiveDescription, cfg.Create.AWSVaultName)
+	result := b.Create(cfg.Create.Files, cfg.Create.GetBlacklist(), cfg.Create.AWSArchiveDescription, cfg.Create.AWSVaultName)
 
 	if result.Error != nil {
 		LogError("Could not upload backup. Error: %v", result.Error)
@@ -44,6 +45,12 @@ func (a *actionCreate) Validate(cfg *config.Config) {
 		cfg.Create.Fail("No file given!")
 	}
 
+	for _, curExpr := range cfg.Create.Blacklist {
+		if _, err := regexp.Compile(curExpr); err != nil {
+			cfg.Create.Fail(fmt.Sprintf(`Blacklist expression is invalid: "%s" Cause: %v`, curExpr, err))
+		}
+	}
+
 	if !isValidPartSize(cfg.Create.AWSPartSize) {
 		cfg.Create.Fail("The part size is not valid. Valid sizes are: %+v", validPartSizes)
 	}

config/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/alexflint/go-arg"
 	"os"
+	"regexp"
 	"time"
 )
 
@@ -34,6 +35,7 @@ type CreateConfig struct {
 
 	AWSVaultName string   `arg:"positional,env:AWS_VAULT_NAME,help:The name of the glacier vault."`
 	Files        []string `arg:"positional,env:FILE,help:The file or folder to backup."`
+	Blacklist    []string `arg:"-b,separate,env:BLACKLIST,help:Regular expressions of files that should be excluded."`
 
 	AWSPartSize           int    `arg:"--aws-part-size,env:AWS_PART_SIZE,help:The size of each part (except the last) in MiB."`
 	AWSArchiveDescription string `arg:"-d,env:AWS_ARCHIVE_DESC,help:The description of the archive."`
@@ -122,7 +124,7 @@ func NewConfig() *Config {
 			DatabaseConfig: DatabaseConfig{
 				Database: DefaultDatabase,
 			},
-			AWSPartSize:  1024 * 1024, //1MB chunk
+			AWSPartSize:  1, //1MB chunk
 			SavePassword: false,
 		}
 
@@ -185,6 +187,16 @@ func NewConfig() *Config {
 	return cfg
 }
 
+func (c *CreateConfig) GetBlacklist() []*regexp.Regexp {
+	var result []*regexp.Regexp
+
+	for _, curEntry := range c.Blacklist {
+		result = append(result, regexp.MustCompile(curEntry))
+	}
+
+	return result
+}
+
 func (c *CreateConfig) Fail(format string, args ...interface{}) {
 	failInternal(c.argParser, format, args...)
 }