Add 'ignore' flag

Via 'ignore' flag, you can exclude local files and keep remote objects stale that are matched with regexp.

Usage example:
```
s3deploy \
    -source=public/ \
    -bucket=example.com \
    -region=eu-west-1 \
    -key=$AWS_ACCESS_KEY_ID \
    -secret=$AWS_SECRET_ACCESS_KEY \
    -ignore="^unsynced-dir/.*$" \
    -v
```

Number of ignored items is not added to any stats count.

Author: satotake

README.md

@@ -42,6 +42,8 @@ Usage of ./s3deploy:
   -force
     	upload even if the etags match
   -h	help
+  -ignore string
+    	regexp pattern for ignoring files
   -key string
     	access key ID for AWS
   -max-delete int

lib/config.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 )
 
@@ -42,6 +43,8 @@ type Config struct {
 	Silent          bool
 	Force           bool
 	Try             bool
+	Ignore          string
+	IgnoreRE        *regexp.Regexp // compiled version of Ignore
 
 	// CLI state
 	PrintVersion bool
@@ -73,6 +76,7 @@ func flagsToConfig(f *flag.FlagSet) (*Config, error) {
 	f.BoolVar(&cfg.PublicReadACL, "public-access", false, "DEPRECATED: please set -acl='public-read'")
 	f.StringVar(&cfg.ACL, "acl", "", "provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default \"private\")")
 	f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match")
+	f.StringVar(&cfg.Ignore, "ignore", "", "regexp pattern for ignoring files")
 	f.BoolVar(&cfg.Try, "try", false, "trial run, no remote updates")
 	f.BoolVar(&cfg.Verbose, "v", false, "enable verbose logging")
 	f.BoolVar(&cfg.Silent, "quiet", false, "enable silent mode")
@@ -105,5 +109,32 @@ func (cfg *Config) check() error {
 		return errors.New("you passed a value for the flags public-access and acl, which is not supported. the public-access flag is deprecated. please use the acl flag moving forward")
 	}
 
+	if cfg.Ignore != "" {
+		re, err := regexp.Compile(cfg.Ignore)
+		if err != nil {
+			return errors.New("cannot compile 'ignore' flag pattern " + err.Error())
+		}
+		cfg.IgnoreRE = re
+	}
+
 	return nil
 }
+
+func (cfg *Config) shouldIgnoreLocal(key string) bool {
+	if cfg.Ignore == "" {
+		return false
+	}
+
+	return cfg.IgnoreRE.MatchString(key)
+}
+
+func (cfg *Config) shouldIgnoreRemote(key string) bool {
+	if cfg.Ignore == "" {
+		return false
+	}
+
+	sub := key[len(cfg.BucketPath):]
+	sub = strings.TrimPrefix(sub, "/")
+
+	return cfg.IgnoreRE.MatchString(sub)
+}

lib/config_test.go

@@ -28,6 +28,7 @@ func TestFlagsToConfig(t *testing.T) {
 		"-region=myregion",
 		"-source=mysource",
 		"-distribution-id=mydistro",
+		"-ignore=^ignored-prefix.*",
 		"-try=true",
 	}
 
@@ -47,6 +48,7 @@ func TestFlagsToConfig(t *testing.T) {
 	assert.Equal(true, cfg.Try)
 	assert.Equal("myregion", cfg.RegionName)
 	assert.Equal("mydistro", cfg.CDNDistributionID)
+	assert.Equal("^ignored-prefix.*", cfg.Ignore)
 }
 
 func TestSetAclAndPublicAccessFlag(t *testing.T) {
@@ -66,3 +68,61 @@ func TestSetAclAndPublicAccessFlag(t *testing.T) {
 	assert.Error(check_err)
 	assert.Contains(check_err.Error(), "you passed a value for the flags public-access and acl")
 }
+
+func TestIgnoreFlagError(t *testing.T) {
+	assert := require.New(t)
+	flags := flag.NewFlagSet("test", flag.PanicOnError)
+	args := []string{
+		"-bucket=mybucket",
+		"-ignore=((INVALID_PATTERN",
+	}
+
+	cfg, err := flagsToConfig(flags)
+	assert.NoError(err)
+	assert.NoError(flags.Parse(args))
+
+	check_err := cfg.check()
+	assert.Error(check_err)
+	assert.Contains(check_err.Error(), "cannot compile 'ignore' flag pattern")
+}
+
+func TestShouldIgnore(t *testing.T) {
+	assert := require.New(t)
+
+	flags_default := flag.NewFlagSet("test", flag.PanicOnError)
+	flags_ignore := flag.NewFlagSet("test", flag.PanicOnError)
+
+	args_default := []string{
+		"-bucket=mybucket",
+		"-path=my/path",
+	}
+	args_ignore := []string{
+		"-bucket=mybucket",
+		"-path=my/path",
+		"-ignore=^ignored-prefix.*",
+	}
+
+	cfg_default, _ := flagsToConfig(flags_default)
+	cfg_ignore, _ := flagsToConfig(flags_ignore)
+
+	flags_default.Parse(args_default)
+	flags_ignore.Parse(args_ignore)
+
+	check_err_default := cfg_default.check()
+	check_err_ignore := cfg_ignore.check()
+
+	assert.NoError(check_err_default)
+	assert.NoError(check_err_ignore)
+
+	assert.False(cfg_default.shouldIgnoreLocal("any"))
+	assert.False(cfg_default.shouldIgnoreLocal("ignored-prefix/file.txt"))
+
+	assert.False(cfg_ignore.shouldIgnoreLocal("any"))
+	assert.True(cfg_ignore.shouldIgnoreLocal("ignored-prefix/file.txt"))
+
+	assert.False(cfg_default.shouldIgnoreRemote("my/path/any"))
+	assert.False(cfg_default.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"))
+
+	assert.False(cfg_ignore.shouldIgnoreRemote("my/path/any"))
+	assert.True(cfg_ignore.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"))
+}

lib/deployer.go

@@ -233,7 +233,12 @@ func (d *Deployer) plan(ctx context.Context) error {
 	close(d.filesToUpload)
 
 	// any remote files not found locally should be removed:
+	// except for ignored files
 	for key := range remoteFiles {
+		if d.cfg.shouldIgnoreRemote(key) {
+			fmt.Fprintf(d.outv, "%s ignored …\n", key)
+			continue
+		}
 		d.enqueueDelete(key)
 	}
 
@@ -273,6 +278,11 @@ func (d *Deployer) walk(ctx context.Context, basePath string, files chan<- *osFi
 		if err != nil {
 			return err
 		}
+
+		if d.cfg.shouldIgnoreLocal(rel) {
+			return nil
+		}
+
 		f, err := newOSFile(d.cfg.conf.Routes, d.cfg.BucketPath, rel, abs, info)
 		if err != nil {
 			return err

lib/deployer_test.go

@@ -103,6 +103,45 @@ func TestDeployForce(t *testing.T) {
 	assert.Equal("Deleted 1 of 1, uploaded 4, skipped 0 (100% changed)", stats.Summary())
 }
 
+func TestDeployWitIgnorePattern(t *testing.T) {
+	assert := require.New(t)
+	root := "my/path"
+	re := `^(main\.css|deleteme\.txt)$`
+
+	store, m := newTestStore(0, root)
+	source := testSourcePath()
+	configFile := filepath.Join(source, ".s3deploy.yml")
+
+	cfg := &Config{
+		BucketName: "example.com",
+		RegionName: "eu-west-1",
+		ConfigFile: configFile,
+		BucketPath: root,
+		MaxDelete:  300,
+		Silent:     false,
+		SourcePath: source,
+		baseStore:  store,
+		Ignore:     re,
+	}
+
+	prevCss := m["my/path/main.css"]
+	prevTag := prevCss.ETag()
+
+	stats, err := Deploy(cfg)
+	assert.NoError(err)
+	assert.Equal("Deleted 0 of 0, uploaded 2, skipped 1 (67% changed)", stats.Summary())
+	assertKeys(t, m,
+		"my/path/.s3deploy.yml",
+		"my/path/index.html",
+		"my/path/ab.txt",
+		"my/path/deleteme.txt", // ignored: stale
+		"my/path/main.css",     // ignored: not updated
+	)
+	mainCss := m["my/path/main.css"]
+	assert.Equal(mainCss.ETag(), prevTag)
+
+}
+
 func TestDeploySourceNotFound(t *testing.T) {
 	assert := require.New(t)
 	store, _ := newTestStore(0, "")