fix: replciation issue local log and seal corruption (#34)

When replicating the remote log file and the remote seal to local disc
the files must be opened in O_TRUNC mode.

The single corner case where this happens required file system based
tests and we only have those in veracity.

Follow on work required if we want those tests to exist in this repo.

AB#10163

Co-authored-by: Robin Bryce <[email protected]>

Author: robinbryce

massifs/localmassifreader.go

@@ -23,7 +23,11 @@ type DirResolver interface {
 	ResolveSealDir(tenantIdentityOrLocalPath string) (string, error)
 }
 type WriteAppendOpener interface {
+	// Open ensures the named file exists and is writable. Writes are appended to any existing content.
 	Open(string) (io.WriteCloser, error)
+	// Create ensures the named file exists, is empty and is writable
+	// If the named file already exists it is truncated
+	Create(string) (io.WriteCloser, error)
 }
 
 type VerifiedContextReader interface {
@@ -183,27 +187,28 @@ func (r *LocalReader) VerifyContext(
 
 // ReplaceVerifiedContext writes the content from the verified remote to the
 // local replica.  is the callers responsibility to ensure the context was
-// verified, and that the writeOpener opens the file in append mode if it
+// verified, and that the writeOpener opens the file in *truncate* mode if it
 // already exists.
 func (r *LocalReader) ReplaceVerifiedContext(
 	vc *VerifiedContext, writeOpener WriteAppendOpener,
 ) error {
 
+	// We are always provided the full massif data, not a delta. So the open
+	// mode is O_TRUNC (empty the file if it exists).  The caller is responsible
+	// for checking the local is consistent with the remote before replacing.
 	logFilename := r.GetMassifLocalPath(vc.TenantIdentity, vc.Start.MassifIndex)
-	err := writeAll(writeOpener, logFilename, vc.Data)
+	err := writeAll(writeOpener.Create, logFilename, vc.Data)
 	if err != nil {
 		return err
 	}
 
 	sealFilename := r.GetSealLocalPath(vc.TenantIdentity, vc.Start.MassifIndex)
-	if err != nil {
-		return err
-	}
+
 	sealBytes, err := vc.Sign1Message.MarshalCBOR()
 	if err != nil {
 		return err
 	}
-	err = writeAll(writeOpener, sealFilename, sealBytes)
+	err = writeAll(writeOpener.Create, sealFilename, sealBytes)
 	if err != nil {
 		return err
 	}
@@ -401,8 +406,10 @@ func copyCachedMassif(cached *MassifContext) MassifContext {
 	return mc
 }
 
-func writeAll(wo WriteAppendOpener, filename string, data []byte) error {
-	f, err := wo.Open(filename)
+type WriteOpener func(name string) (io.WriteCloser, error)
+
+func writeAll(wo WriteOpener, filename string, data []byte) error {
+	f, err := wo(filename)
 	if err != nil {
 		return err