Merge pull request #1549 from 0chain/fix/notfound-ref

fix ref not found and add log for copy error

Author: dabasov

zboxcore/allocationchange/deletefile.go

@@ -13,6 +13,8 @@ type DeleteFileChange struct {
 	FileMetaRef fileref.RefEntity
 }
 
+var ErrRefNotFound = errors.New("ref_not_found", "Ref not found")
+
 func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]string) (err error) {
 
 	if ch.FileMetaRef.GetPath() == "/" {
@@ -39,7 +41,7 @@ func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str
 		}
 
 		if !found {
-			err = errors.New("invalid_reference_path", "Invalid reference path from the blobber")
+			err = ErrRefNotFound
 			return
 		}
 	}

zboxcore/sdk/allocation.go

@@ -946,7 +946,7 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul
 				operation, newConnectionID, err = NewUploadOperation(mo.ctx, op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, true, op.IsWebstreaming, op.IsRepair, op.DownloadFile, op.StreamUpload, op.Opts...)
 
 			case constants.FileOperationCreateDir:
-				operation = NewDirOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)
+				operation = NewDirOperation(op.RemotePath, op.FileMeta.CustomMeta, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)
 
 			default:
 				return errors.New("invalid_operation", "Operation is not valid")

zboxcore/sdk/chunked_upload_form_builder.go

@@ -88,6 +88,7 @@ func (b *chunkedUploadFormBuilder) Build(
 		Size:              shardSize,
 		EncryptedKeyPoint: encryptedKeyPoint,
 		EncryptedKey:      encryptedKey,
+		CustomMeta:        fileMeta.CustomMeta,
 	}
 
 	for i := 0; i < numBodies; i++ {

zboxcore/sdk/chunked_upload_model.go

@@ -119,6 +119,8 @@ type FileMeta struct {
 	RemoteName string
 	// RemotePath remote path
 	RemotePath string
+	// CustomMeta custom meta data
+	CustomMeta string
 }
 
 // FileID generate id of progress on local cache

zboxcore/sdk/commitworker.go

@@ -193,10 +193,13 @@ func (commitreq *CommitRequest) processCommit() {
 	for _, change := range commitreq.changes {
 		err = change.ProcessChange(rootRef, fileIDMeta)
 		if err != nil {
-			commitreq.result = ErrorCommitResult(err.Error())
-			return
+			if !errors.Is(err, allocationchange.ErrRefNotFound) {
+				commitreq.result = ErrorCommitResult(err.Error())
+				return
+			}
+		} else {
+			size += change.GetSize()
 		}
-		size += change.GetSize()
 	}
 	rootRef.CalculateHash()
 	var chainHash string

zboxcore/sdk/copyworker.go

@@ -178,7 +178,7 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, []error) {
 			refEntity, err := req.copyBlobberObject(req.blobbers[blobberIdx], blobberIdx)
 			if err != nil {
 				blobberErrors[blobberIdx] = err
-				l.Logger.Error(err.Error())
+				l.Logger.Debug(err.Error())
 				return
 			}
 			objectTreeRefs[blobberIdx] = refEntity
@@ -324,6 +324,7 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f
 	objectTreeRefs, blobberErrors := cR.ProcessWithBlobbers()
 
 	if !cR.isConsensusOk() {
+		l.Logger.Error("copy failed: ", cR.remotefilepath, cR.destPath)
 		err := zboxutil.MajorError(blobberErrors)
 		if err != nil {
 			return nil, cR.copyMask, errors.New("copy_failed", fmt.Sprintf("Copy failed. %s", err.Error()))

zboxcore/sdk/dirworker.go

@@ -43,6 +43,7 @@ type DirRequest struct {
 	connectionID  string
 	timestamp     int64
 	alreadyExists map[uint64]bool
+	customMeta    string
 	Consensus
 }
 
@@ -177,6 +178,13 @@ func (req *DirRequest) createDirInBlobber(blobber *blockchain.StorageNode, pos u
 		return err, false
 	}
 
+	if req.customMeta != "" {
+		err = formWriter.WriteField("custom_meta", req.customMeta)
+		if err != nil {
+			return err, false
+		}
+	}
+
 	formWriter.Close()
 	httpreq, err := zboxutil.NewCreateDirRequest(blobber.Baseurl, req.allocationID, req.allocationTx, body)
 	if err != nil {
@@ -238,12 +246,12 @@ func (req *DirRequest) createDirInBlobber(blobber *blockchain.StorageNode, pos u
 			latestStatusCode = resp.StatusCode
 
 			msg = string(respBody)
-			l.Logger.Error(blobber.Baseurl, " Response: ", msg)
 			if strings.Contains(msg, DirectoryExists) {
 				req.Consensus.Done()
 				alreadyExists = true
 				return
 			}
+			l.Logger.Error(blobber.Baseurl, " Response: ", msg)
 
 			err = errors.New("response_error", msg)
 			return
@@ -271,6 +279,7 @@ type DirOperation struct {
 	ctxCncl       context.CancelFunc
 	dirMask       zboxutil.Uint128
 	maskMU        *sync.Mutex
+	customMeta    string
 	alreadyExists map[uint64]bool
 
 	Consensus
@@ -290,6 +299,7 @@ func (dirOp *DirOperation) Process(allocObj *Allocation, connectionID string) ([
 		mu:            dirOp.maskMU,
 		wg:            &sync.WaitGroup{},
 		alreadyExists: make(map[uint64]bool),
+		customMeta:    dirOp.customMeta,
 	}
 	dR.Consensus = Consensus{
 		RWMutex:         &sync.RWMutex{},
@@ -347,13 +357,14 @@ func (dirOp *DirOperation) Error(allocObj *Allocation, consensus int, err error)
 
 }
 
-func NewDirOperation(remotePath string, dirMask zboxutil.Uint128, maskMU *sync.Mutex, consensusTh int, fullConsensus int, ctx context.Context) *DirOperation {
+func NewDirOperation(remotePath, customMeta string, dirMask zboxutil.Uint128, maskMU *sync.Mutex, consensusTh int, fullConsensus int, ctx context.Context) *DirOperation {
 	dirOp := &DirOperation{}
 	dirOp.remotePath = zboxutil.RemoteClean(remotePath)
 	dirOp.dirMask = dirMask
 	dirOp.maskMU = maskMU
 	dirOp.consensusThresh = consensusTh
 	dirOp.fullconsensus = fullConsensus
+	dirOp.customMeta = customMeta
 	dirOp.ctx, dirOp.ctxCncl = context.WithCancel(ctx)
 	dirOp.alreadyExists = make(map[uint64]bool)
 	return dirOp

zboxcore/sdk/downloadworker.go

@@ -1207,7 +1207,7 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse)
 			fRef.ActualFileHashSignature+fRef.ValidationRoot,
 		)
 		if err != nil {
-			l.Logger.Error(err)
+			l.Logger.Error(err, "allocOwnerPubKey: ", req.allocOwnerPubKey, " validationRootSignature: ", fRef.ValidationRootSignature, " actualFileHashSignature: ", fRef.ActualFileHashSignature, " validationRoot: ", fRef.ValidationRoot)
 			continue
 		}
 		if !isValid {

zboxcore/sdk/filerefsworker.go

@@ -227,6 +227,7 @@ type SimilarField struct {
 	MimeType            string `json:"mimetype"`
 	ActualThumbnailSize int64  `json:"actual_thumbnail_size"`
 	ActualThumbnailHash string `json:"actual_thumbnail_hash"`
+	CustomMeta          string `json:"custom_meta"`
 }
 
 type RecentlyAddedRefRequest struct {