feat: encrypt large files with convergent encryption (#25)

* chore: add gitignore file

Signed-off-by: François SAMIN <[email protected]>

* chore: go module

Signed-off-by: François SAMIN <[email protected]>

* feat: encrypt large files with convergent encryption

including:
* ChunksWriter and ChunksReader
* ConvergentKey

Signed-off-by: François SAMIN <[email protected]>

* test: chunksRead, dhunksWriter and convergentKey

Signed-off-by: François SAMIN <[email protected]>

* feat: sequential key and deduplication with locator

Signed-off-by: François SAMIN <[email protected]>

* fix: lint

* chore: golang version in travis file

Signed-off-by: francois  samin <[email protected]>

* chore: avoid travis-ci to get out of memory

Signed-off-by: francois  samin <[email protected]>

* feat: new high-level feature in 'convergent' package

Signed-off-by: francois  samin <[email protected]>

* refactor: rename SequentialKey with SequenceKey

Signed-off-by: francois  samin <[email protected]>

* chore: update golangci-lint version

Signed-off-by: francois  samin <[email protected]>

* fix code review

* feat: Locator() should be public

* feat: try to close the destination writer

* Apply suggestions from code review

Co-authored-by: Thomas Schaffer <[email protected]>

* fix

* Update symmecrypt.go

Co-authored-by: Thomas Schaffer <[email protected]>

* wip

* fix: move  EncryptPipe + DecryptPipe in stream package

Signed-off-by: francois  samin <[email protected]>

* fix: typo

Signed-off-by: francois  samin <[email protected]>

* fix: avoid io.ErrShortWrite

Signed-off-by: francois  samin <[email protected]>

* fix: avoid io.ErrShortWrite

Signed-off-by: francois  samin <[email protected]>

* fix: avoid io.ErrShortWrite

Signed-off-by: francois  samin <[email protected]>

* fix: cr

Signed-off-by: francois  samin <[email protected]>

* Distinguish seal handling errors

Signed-off-by: Thomas Schaffer <[email protected]>

* Uniquely identify sealed key errors from keyloader package

Signed-off-by: Thomas Schaffer <[email protected]>

Co-authored-by: Thomas Schaffer <[email protected]>
Co-authored-by: Thomas Schaffer <[email protected]>

Author: fsamin

.gitignore

@@ -0,0 +1,3 @@
+Test*
+vendor/
+cmd/symmecrypt/symmecrypt

.travis.yml

@@ -15,7 +15,7 @@ notifications:
 # build and immediately stop. It's sorta like having set -e enabled in bash.
 # Make sure golangci-lint is vendored.
 before_script:
-  - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.25.0
+  - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
 
 # script always runs to completion (set +e). If we have linter issues AND a
 # failing test, we want to see both. Configure golangci-lint with a

ciphers/hmac/hmac.go

@@ -34,6 +34,10 @@ func init() {
 
 type hmacFactory struct{}
 
+func (f hmacFactory) KeyLen() int {
+	return KeyLen
+}
+
 func (f hmacFactory) NewKey(s string) (symmecrypt.Key, error) {
 	k, err := symutils.RawKey([]byte(s), KeyLen)
 	if err != nil {
@@ -50,6 +54,16 @@ func (f hmacFactory) NewRandomKey() (symmecrypt.Key, error) {
 	return Key(b), nil
 }
 
+func (f hmacFactory) NewSequenceKey(s string) (symmecrypt.Key, error) {
+	// the hmac cipher doesnt use a nonce, so a sequence key == a regular key
+	return f.NewKey(s)
+}
+
+func (f hmacFactory) NewRandomSequenceKey() (symmecrypt.Key, error) {
+	// the hmac cipher doesnt use a nonce, so a sequence key == a regular key
+	return f.NewRandomKey()
+}
+
 // Key is a simple key which uses plain data + HMAC-sha512 for authentication
 type Key []byte
 

ciphers/hmac/hmac_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestEncrypt(t *testing.T) {
-
+	
 	text := []byte("foobar")
 
 	k, err := symmecrypt.NewRandomKey(CipherName)

cmd/symmecrypt/main.go

@@ -73,7 +73,7 @@ func readKey() error {
 	for _, encodedKey := range keys {
 		plain, err := base64.StdEncoding.DecodeString(encodedKey)
 		if err != nil {
-			return fmt.Errorf("Invalid base64 encryption key: %s", err)
+			return fmt.Errorf("Invalid base64 encryption key: %w", err)
 		}
 		keyList = append(keyList, configstore.NewItem("encryption-key", string(plain), 1))
 	}
@@ -96,11 +96,11 @@ func main() {
 		}
 		key, err := keyloader.GenerateKey(*newEncryptionCipher, *keyIdentifier, false, time.Now())
 		if err != nil {
-			log.Fatalf("error: unable to generate key: %s", err)
+			log.Fatalf("error: unable to generate key: %v", err)
 		}
 		j, err := json.Marshal(key)
 		if err != nil {
-			log.Fatalf("error: unable to generate key: %s", err)
+			log.Fatalf("error: unable to generate key: %v", err)
 		}
 		newKey := string(j)
 		if *useBase64 {
@@ -120,7 +120,7 @@ func main() {
 			k, err = keyloader.LoadSingleKey()
 		}
 		if err != nil {
-			log.Fatalf("error: failed to instantiate key: %s", err)
+			log.Fatalf("error: failed to instantiate key: %v", err)
 		}
 		dataStr := readSecret()
 		extra := [][]byte{}
@@ -129,7 +129,7 @@ func main() {
 		}
 		b, err := k.Encrypt([]byte(dataStr), extra...)
 		if err != nil {
-			log.Fatalf("error: failed to encrypt: %s", err)
+			log.Fatalf("error: failed to encrypt: %v", err)
 		}
 		outputStr := string(b)
 		if *useBase64 {
@@ -149,13 +149,13 @@ func main() {
 			k, err = keyloader.LoadSingleKey()
 		}
 		if err != nil {
-			log.Fatalf("error: failed to instantiate key: %s", err)
+			log.Fatalf("error: failed to instantiate key: %v", err)
 		}
 		dataStr := readSecret()
 		if *useBase64 {
 			dataRaw, err := base64.StdEncoding.DecodeString(dataStr)
 			if err != nil {
-				log.Fatalf("error: failed to decode base64: %s", err)
+				log.Fatalf("error: failed to decode base64: %v", err)
 			}
 			dataStr = string(dataRaw)
 		}
@@ -165,7 +165,7 @@ func main() {
 		}
 		b, err := k.Decrypt([]byte(dataStr), extra...)
 		if err != nil {
-			log.Fatalf("error: failed to decrypt: %s", err)
+			log.Fatalf("error: failed to decrypt: %v", err)
 		}
 		fmt.Print(string(b))
 	}
@@ -177,7 +177,7 @@ func readSecret() string {
 		os.Exit(0)
 	}
 	if err != nil {
-		log.Fatalf("error: failed to read input: %s", err)
+		log.Fatalf("error: failed to read input: %v", err)
 	}
 	return string(b)
 }