Merge pull request #109 from gregdhill/chunk

chunk get if stored data is > chunk size

CHANGELOG.md

@@ -1,4 +1,9 @@
 # [Monax Hoard](https://github.com/monax/hoard) Changelog
+## [8.2.2] - 2020-03-25
+### Fixed
+- Chunk plaintext out if data too big
+
+
 ## [8.2.1] - 2020-03-23
 ### Fixed
 - Convergent encryption nonce is now specified
@@ -192,6 +197,7 @@ This is the first Hoard open source release and includes:
 	- Hoar-Daemon hoard
 	- Hoar-Control hoarctl CLI
 
+[8.2.2]: https://github.com/monax/hoard/compare/v8.2.1...v8.2.2
 [8.2.1]: https://github.com/monax/hoard/compare/v8.2.0...v8.2.1
 [8.2.0]: https://github.com/monax/hoard/compare/v8.1.0...v8.2.0
 [8.1.0]: https://github.com/monax/hoard/compare/v8.0.2...v8.1.0

NOTES.md

@@ -1,3 +1,3 @@
 ### Fixed
-- Convergent encryption nonce is now specified
+- Chunk plaintext out if data too big
 

project/history.go

@@ -40,6 +40,10 @@ func FullVersion() string {
 // release tagging script: ./scripts/tag_release.sh
 var History relic.ImmutableHistory = relic.NewHistory("Monax Hoard", "https://github.com/monax/hoard").
 	MustDeclareReleases(
+		"8.2.2 - 2020-03-25",
+		`### Fixed
+- Chunk plaintext out if data too big
+`,
 		"8.2.1 - 2020-03-23",
 		`### Fixed
 - Convergent encryption nonce is now specified

service.go

@@ -51,7 +51,7 @@ func (service *Service) Get(srv api.Cleartext_GetServer) error {
 			return err
 		}
 
-		if err = SendPlaintext(data, srv, ref.GetVersion()); err != nil {
+		if err = SendPlaintext(data, service.chunkSize, srv, ref.GetVersion()); err != nil {
 			return err
 		}
 	}
@@ -150,7 +150,7 @@ func (service *Service) Decrypt(srv api.Encryption_DecryptServer) error {
 			return err
 		}
 
-		if err = SendPlaintext(data, srv, refAndCiphertext.Reference.GetVersion()); err != nil {
+		if err = SendPlaintext(data, service.chunkSize, srv, refAndCiphertext.Reference.GetVersion()); err != nil {
 			return err
 		}
 	}
@@ -326,7 +326,7 @@ func (service *Service) UnsealGet(grt *grant.Grant, srv api.Grant_UnsealGetServe
 			return err
 		}
 
-		if err = SendPlaintext(data, srv, ref.GetVersion()); err != nil {
+		if err = SendPlaintext(data, service.chunkSize, srv, ref.GetVersion()); err != nil {
 			return err
 		}
 	}

service_test.go

@@ -36,7 +36,7 @@ func writeUIntBE(buffer []byte, value, offset, byteLength int64) error {
 }
 
 func TestService(t *testing.T) {
-	chunkSize := 67
+	chunkSize := 1024 * 1024
 	salt, err := encryption.NewNonce(encryption.NonceSize)
 	assert.NoError(t, err)
 	secret, err := encryption.DeriveSecretKey([]byte("shhhh"), salt)
@@ -136,6 +136,26 @@ func TestService(t *testing.T) {
 				require.Equal(t, meta, head)
 				require.Equal(t, data, rest)
 			})
+
+			t.Run("ChunkLarge", func(t *testing.T) {
+				size := 1024 * 1024 * 10
+				ref, err := hrd.Put(make([]byte, size), nil)
+				require.NoError(t, err)
+
+				client := api.NewCleartextClient(conn)
+				getStream, err := client.Get(ctx)
+				require.NoError(t, err)
+
+				err = getStream.Send(ref)
+				require.NoError(t, err)
+				err = getStream.CloseSend()
+				require.NoError(t, err)
+
+				plaintext, err := ReceiveAllPlaintexts(getStream)
+				require.NoError(t, err)
+
+				require.Equal(t, size, len(plaintext.GetBody()))
+			})
 		})
 
 		t.Run("Grants", func(t *testing.T) {

streaming.go

@@ -15,7 +15,7 @@ type PlaintextSender interface {
 }
 
 // SendPlaintext gets the plaintext for a given reference and sends it to the client
-func SendPlaintext(data []byte, srv PlaintextSender, version int32) error {
+func SendPlaintext(data []byte, chunkSize int, srv PlaintextSender, version int32) error {
 	if version == defaultRefVersionForHeader {
 		head := new(api.Header)
 		err := proto.Unmarshal(data, head)
@@ -25,6 +25,14 @@ func SendPlaintext(data []byte, srv PlaintextSender, version int32) error {
 		return srv.Send(&api.Plaintext{Head: head})
 	}
 
+	if len(data) > chunkSize {
+		return sendChunks(data, chunkSize, func(chunk []byte) error {
+			return srv.Send(&api.Plaintext{
+				Body: chunk,
+			})
+		})
+	}
+
 	return srv.Send(&api.Plaintext{Body: data})
 }
 
@@ -253,3 +261,18 @@ func ReadStream(receiver func() (interface{}, error)) (interface{}, error) {
 		}
 	}
 }
+
+func sendChunks(data []byte, chunkSize int, sender func(chunk []byte) error) error {
+	var err error
+	for i := 0; i < len(data); i += chunkSize {
+		if i+chunkSize > len(data) {
+			err = sender(data[i:])
+		} else {
+			err = sender(data[i : i+chunkSize])
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}