Timestamp changes (#1210)

* minor changes to chalk in zus overview

* check start time

* parse form in parallel

* fix err msg

* don't parse for delete

* valid sig fix

* check form

* fixed configs

* rmv log

* change pending check

* use buf

* rmv log

---------

Co-authored-by: Kishan Dhakan <[email protected]>
Co-authored-by: Harshit Mehndiratta <[email protected]>
Co-authored-by: dabasov <[email protected]>

README.md

@@ -32,13 +32,11 @@ Users can also add their own servers to the network to operate in a hybrid cloud
 
 The [privacy protocol](https://zus.network/build) from Züs is unique where a user can easily share their encrypted data with their business partners, friends, and family through a proxy key sharing protocol, where the key is given to the providers, and they re-encrypt the data using the proxy key so that only the recipient can decrypt it with their private key.
 
-Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a zero upfront cost permanent storage solution for NFT artists.
+Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a high-performance story-telling storage solution for NFT artists.
 
 Other apps are [Bolt](https://bolt.holdings/), a wallet that is very secure with air-gapped 2FA split-key protocol to prevent hacks from compromising your digital assets, and it enables you to stake and earn from the storage providers; [Atlus](https://atlus.cloud/), a blockchain explorer and [Chimney](https://demo.chimney.software/), which allows anyone to join the network and earn using their server or by just renting one, with no prior knowledge required.
 
 
-
-
 ## Initial Setup
 
 ### Required OS and Software Dependencies

code/go/0chain.net/blobbercore/allocation/allocationchange.go

@@ -217,7 +217,6 @@ func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocatio
 			return err
 		}
 	}
-	logging.Logger.Info("ApplyChanges", zap.Any("rootRef", rootRef))
 	_, err = rootRef.CalculateHash(ctx, true)
 	return err
 }

code/go/0chain.net/blobbercore/allocation/entity.go

@@ -48,13 +48,14 @@ type Allocation struct {
 	RepairerID     string           `gorm:"column:repairer_id;size:64;not null"`
 	Expiration     common.Timestamp `gorm:"column:expiration_date;not null"`
 	// AllocationRoot allcation_root of last write_marker
-	AllocationRoot   string        `gorm:"column:allocation_root;size:64;not null;default:''"`
-	FileMetaRoot     string        `gorm:"column:file_meta_root;size:64;not null;default:''"`
-	BlobberSize      int64         `gorm:"column:blobber_size;not null;default:0"`
-	BlobberSizeUsed  int64         `gorm:"column:blobber_size_used;not null;default:0"`
-	LatestRedeemedWM string        `gorm:"column:latest_redeemed_write_marker;size:64"`
-	IsRedeemRequired bool          `gorm:"column:is_redeem_required"`
-	TimeUnit         time.Duration `gorm:"column:time_unit;not null;default:172800000000000"`
+	AllocationRoot   string           `gorm:"column:allocation_root;size:64;not null;default:''"`
+	FileMetaRoot     string           `gorm:"column:file_meta_root;size:64;not null;default:''"`
+	BlobberSize      int64            `gorm:"column:blobber_size;not null;default:0"`
+	BlobberSizeUsed  int64            `gorm:"column:blobber_size_used;not null;default:0"`
+	LatestRedeemedWM string           `gorm:"column:latest_redeemed_write_marker;size:64"`
+	IsRedeemRequired bool             `gorm:"column:is_redeem_required"`
+	TimeUnit         time.Duration    `gorm:"column:time_unit;not null;default:172800000000000"`
+	StartTime        common.Timestamp `gorm:"column:start_time;not null"`
 	// Ending and cleaning
 	CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"`
 	Finalized bool `gorm:"column:finalized;not null;default:false"`

code/go/0chain.net/blobbercore/allocation/protocol.go

@@ -108,6 +108,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc
 	a.Finalized = sa.Finalized
 	a.TimeUnit = sa.TimeUnit
 	a.FileOptions = sa.FileOptions
+	a.StartTime = sa.StartTime
 
 	m := map[string]interface{}{
 		"allocation_id":  a.ID,

code/go/0chain.net/blobbercore/filestore/storage.go

@@ -83,8 +83,8 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i
 	if err != nil {
 		return nil, common.NewError("file_seek_error", err.Error())
 	}
-
-	writtenSize, err := io.Copy(f, infile)
+	buf := make([]byte, BufferSize)
+	writtenSize, err := io.CopyBuffer(f, infile, buf)
 	if err != nil {
 		return nil, common.NewError("file_write_error", err.Error())
 	}

code/go/0chain.net/blobbercore/handler/file_command_upload.go

@@ -50,7 +50,6 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
 	}
 
 	fileChanger := &allocation.UploadFileChanger{}
-
 	uploadMetaString := req.FormValue(UploadMeta)
 	err := json.Unmarshal([]byte(uploadMetaString), fileChanger)
 	if err != nil {

code/go/0chain.net/blobbercore/handler/object_operation_handler.go

@@ -494,19 +494,19 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
 		return nil, common.NewError("invalid_parameters", "Invalid connection id passed")
 	}
 
-	err = checkPendingMarkers(ctx, allocationObj.ID)
-	if err != nil {
-		Logger.Error("Error checking pending markers", zap.Error(err))
-		return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed")
-	}
-
 	// Lock will compete with other CommitWrites and Challenge validation
 	mutex := lock.GetMutex(allocationObj.TableName(), allocationID)
 	mutex.Lock()
 	defer mutex.Unlock()
 
 	elapsedGetLock := time.Since(startTime) - elapsedAllocation
 
+	err = checkPendingMarkers(ctx, allocationObj.ID)
+	if err != nil {
+		Logger.Error("Error checking pending markers", zap.Error(err))
+		return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed")
+	}
+
 	connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID)
 
 	if err != nil {

code/go/0chain.net/blobbercore/readmarker/readmarker.go

@@ -103,8 +103,11 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All
 		return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID")
 	}
 
+	if rm.LatestRM.Timestamp < sa.StartTime {
+		return common.NewError("read_marker_validation_failed", "Readmarker timestamp is before the allocation start time")
+	}
+
 	if rm.LatestRM.Timestamp > sa.Expiration {
-		zLogger.Logger.Error("Readmarker is for an expired allocation", zap.Any("rm", rm))
 		return common.NewError("read_marker_validation_failed", "Readmarker is for an expired allocation")
 	}
 

code/go/0chain.net/blobbercore/writemarker/protocol.go

@@ -81,10 +81,13 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al
 	if clientID == "" || clientID != wme.WM.ClientID || clientID != co.ClientID || co.ClientID != wme.WM.ClientID {
 		return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded")
 	}
+	if wme.WM.Timestamp < dbAllocation.StartTime {
+		return common.NewError("write_marker_validation_failed", "Write Marker timestamp is before the allocation start time")
+	}
 
 	currTime := common.Now()
 	// blobber clock is allowed to be 10 seconds behind the current time
-	if wme.WM.Timestamp > currTime+10 {
+	if wme.WM.Timestamp > currTime+60 {
 		return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future")
 	}
 

code/go/0chain.net/core/transaction/entity.go

@@ -90,6 +90,7 @@ type StorageAllocation struct {
 	TimeUnit       time.Duration        `json:"time_unit"`
 	WritePool      uint64               `json:"write_pool"`
 	FileOptions    uint16               `json:"file_options"`
+	StartTime      common.Timestamp     `json:"start_time"`
 
 	DataShards   int64 `json:"data_shards"`
 	ParityShards int64 `json:"parity_shards"`

config/0chain_blobber.yaml

@@ -24,7 +24,7 @@ max_dirs_files: 50000
 
 # delegate wallet (must be set)
 delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9'
-`# maximum allowed number of stake holders
+# maximum allowed number of stake holders
 num_delegates: 50
 # service charge of the blobber
 service_charge: 0.10

goose/migrations/001_blobber_meta.sql

@@ -89,7 +89,8 @@ CREATE TABLE public.allocations (
     time_unit bigint DEFAULT '172800000000000'::bigint NOT NULL,
     cleaned_up boolean DEFAULT false NOT NULL,
     finalized boolean DEFAULT false NOT NULL,
-    file_options integer DEFAULT 63 NOT NULL
+    file_options integer DEFAULT 63 NOT NULL,
+    start_time bigint NOT NULL
 );