Merge pull request #1185 from 0chain/sprint-july-3

Sprint july 3
0chain · Jul 26, 2023 · 3109e16 · 3109e16
2 parents 54b3c2a + 37b1a6c
commit 3109e16
Show file tree

Hide file tree

Showing 14 changed files with 106 additions and 131 deletions.
diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml
@@ -16,6 +16,8 @@ env:
   BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }}
   VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }}
   DOCKER_CLI_EXPERIMENTAL: enabled
+  BLOBBER_BUILDBASE: blobber_base
+  BLOBBER_BUILD_BASE_REGISTRY: ${{ secrets.BLOBBER_BUILD_BASE_REGISTRY }}
 
 jobs:
   blobber:
@@ -39,7 +41,7 @@ jobs:
           go-version: ^1.20 # The Go version to download (if necessary) and use.
 
       - name: Clone blobber
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
         run: |
@@ -60,6 +62,26 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/[email protected]
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh &&
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build blobber
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
@@ -68,7 +90,7 @@ jobs:
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh
+          ./docker.local/bin/build.blobber.sh
 
   validator:
     runs-on: [self-hosted, arc-runner]
@@ -112,14 +134,34 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/[email protected]
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh 
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build validator
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
           export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.validator.sh
+          ./docker.local/bin/build.validator.sh
 
 
   system-tests:

diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go
@@ -24,25 +24,23 @@ type BCChallengeResponse struct {
 	Challenges []*ChallengeEntity `json:"challenges"`
 }
 
-var lastChallengeTimestamp int
+var lastChallengeRound int64
 
 func syncOpenChallenges(ctx context.Context) {
-	const incrOffset = 20
 	defer func() {
 		if r := recover(); r != nil {
 			logging.Logger.Error("[recover]challenge", zap.Any("err", r))
 		}
 	}()
 
-	offset := 0
 	params := make(map[string]string)
 	params["blobber"] = node.Self.ID
-	params["offset"] = strconv.Itoa(offset)
-	params["limit"] = "20"
-	if lastChallengeTimestamp > 0 {
-		params["from"] = strconv.Itoa(lastChallengeTimestamp)
+
+	params["limit"] = "50"
+	if lastChallengeRound > 0 {
+		params["from"] = strconv.FormatInt(lastChallengeRound, 10)
 	}
-	logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params))
+
 	start := time.Now()
 
 	var downloadElapsed, jsonElapsed time.Duration
@@ -54,6 +52,9 @@ func syncOpenChallenges(ctx context.Context) {
 			return
 		default:
 		}
+
+		logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params))
+
 		var challenges BCChallengeResponse
 		var challengeIDs []string
 		challenges.Challenges = make([]*ChallengeEntity, 0)
@@ -75,13 +76,13 @@ func syncOpenChallenges(ctx context.Context) {
 			break
 		}
 		sort.Slice(challenges.Challenges, func(i, j int) bool {
-			return challenges.Challenges[i].CreatedAt < challenges.Challenges[j].CreatedAt
+			return challenges.Challenges[i].RoundCreatedAt < challenges.Challenges[j].RoundCreatedAt
 		})
 		count += len(challenges.Challenges)
 		for _, c := range challenges.Challenges {
 			challengeIDs = append(challengeIDs, c.ChallengeID)
-			if c.CreatedAt > common.Timestamp(lastChallengeTimestamp) {
-				lastChallengeTimestamp = int(c.CreatedAt)
+			if c.RoundCreatedAt >= lastChallengeRound {
+				lastChallengeRound = c.RoundCreatedAt
 			}
 			toProcessChallenge <- c
 		}
@@ -93,8 +94,6 @@ func syncOpenChallenges(ctx context.Context) {
 		if len(challenges.Challenges) == 0 {
 			break
 		}
-		offset += incrOffset
-		params["offset"] = strconv.Itoa(offset)
 	}
 
 	dbTimeStart := time.Now()
@@ -110,6 +109,11 @@ func syncOpenChallenges(ctx context.Context) {
 
 func validateOnValidators(c *ChallengeEntity) {
 
+	logging.Logger.Info("[challenge]validate: ",
+		zap.Any("challenge", c),
+		zap.String("challenge_id", c.ChallengeID),
+	)
+
 	ctx := datastore.GetStore().CreateTransaction(context.TODO())
 	defer ctx.Done()
 
@@ -119,7 +123,7 @@ func validateOnValidators(c *ChallengeEntity) {
 		logging.Logger.Error("[challengetiming]add: ",
 			zap.String("challenge_id", c.ChallengeID),
 			zap.Error(err))
-		deleteChallenge(int64(c.CreatedAt))
+		deleteChallenge(c.RoundCreatedAt)
 		tx.Rollback()
 	}
 
@@ -150,7 +154,7 @@ func validateOnValidators(c *ChallengeEntity) {
 			zap.Time("created", createdTime),
 			zap.Error(err))
 		//TODO: Should we delete the challenge from map or send it back to the todo channel?
-		deleteChallenge(int64(c.CreatedAt))
+		deleteChallenge(c.RoundCreatedAt)
 		tx.Rollback()
 		return
 	}

diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go
@@ -97,8 +97,9 @@ type ChallengeEntity struct {
 	Timestamp               common.Timestamp      `gorm:"column:timestamp;not null;default:0" json:"timestamp"`
 
 	// This time is taken from Blockchain challenge object.
-	CreatedAt common.Timestamp `gorm:"created_at" json:"created"`
-	UpdatedAt time.Time        `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"`
+	RoundCreatedAt int64            `gorm:"round_created_at" json:"round_created_at"`
+	CreatedAt      common.Timestamp `gorm:"created_at" json:"created"`
+	UpdatedAt      time.Time        `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"`
 }
 
 func (ChallengeEntity) TableName() string {

diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go
@@ -48,7 +48,7 @@ type ChallengeResponse struct {
 func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) {
 	cancellation := time.Now()
 	db := datastore.GetStore().GetTransaction(ctx)
-	deleteChallenge(int64(cr.CreatedAt))
+	deleteChallenge(cr.RoundCreatedAt)
 	cr.Status = Cancelled
 	cr.StatusMessage = errReason.Error()
 	cr.UpdatedAt = cancellation.UTC()

diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go
@@ -81,6 +81,8 @@ func challengeProcessor(ctx context.Context) {
 			return
 
 		case it := <-toProcessChallenge:
+
+			logging.Logger.Info("processing_challenge", zap.Any("challenge_id", it.ChallengeID))
 			if ok := it.createChallenge(); !ok {
 				continue
 			}
@@ -142,7 +144,7 @@ func commitOnChainWorker(ctx context.Context) {
 					}()
 					err := challenge.VerifyChallengeTransaction(txn)
 					if err == nil || err != ErrEntityNotFound {
-						deleteChallenge(int64(challenge.CreatedAt))
+						deleteChallenge(int64(challenge.RoundCreatedAt))
 					}
 				}(&chall)
 			}
@@ -175,11 +177,11 @@ func getBatch(batchSize int) (chall []ChallengeEntity) {
 
 func (it *ChallengeEntity) createChallenge() bool {
 	challengeMapLock.Lock()
-	if _, ok := challengeMap.Get(int64(it.CreatedAt)); ok {
+	if _, ok := challengeMap.Get(it.RoundCreatedAt); ok {
 		challengeMapLock.Unlock()
 		return false
 	}
-	challengeMap.Put(int64(it.CreatedAt), it)
+	challengeMap.Put(it.RoundCreatedAt, it)
 	challengeMapLock.Unlock()
 	return true
 }

diff --git a/docker.local/.dockerignore b/docker.local/.dockerignore
@@ -1,8 +1,13 @@
 passphrase.txt
 logs/
+docs/
 .git
 *.md
 .cache
 awsnet
 docker.local/blobber*
+docker.aws/*
+config/
+docker-clean/
+keys_config
 **/pkg
diff --git a/docker.local/b0docker-compose-debug.yml b/docker.local/b0docker-compose-debug.yml
@@ -14,26 +14,10 @@ services:
       - ./sql_init:/docker-entrypoint-initdb.d
     networks:
       default:
-  postgres-post:
-    image: postgres:14
-    environment:
-      POSTGRES_PORT: 5432
-      POSTGRES_HOST: postgres
-      POSTGRES_USER: postgres
-    volumes:
-      - ../bin:/blobber/bin
-      - ../sql:/blobber/sql
-    command: bash /blobber/bin/postgres-entrypoint.sh
-    links:
-      - postgres:postgres
   validator:
     image: validator
     environment:
       - DOCKER= true
-    depends_on:
-      - postgres-post
-    links:
-      - postgres-post:postgres-post
     volumes:
       - ../config:/validator/config
       - ./blobber${BLOBBER}/data:/validator/data

diff --git a/docker.local/b0docker-compose-github.yml b/docker.local/b0docker-compose-github.yml
@@ -14,18 +14,6 @@ services:
       - ./sql_init:/docker-entrypoint-initdb.d
     networks:
       default:
-  postgres-post:
-    image: postgres:14
-    environment:
-      POSTGRES_PORT: 5432
-      POSTGRES_HOST: postgres
-      POSTGRES_USER: postgres
-    volumes:
-      - ../bin:/blobber/bin
-      - ../sql:/blobber/sql
-    command: bash /blobber/bin/postgres-entrypoint.sh
-    links:
-      - postgres:postgres
 
 networks:
   default:

diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml
@@ -14,27 +14,10 @@ services:
       - ./sql_init:/docker-entrypoint-initdb.d
     networks:
       default:
-  postgres-post:
-    image: postgres:14
-    environment:
-      POSTGRES_PORT: 5432
-      POSTGRES_HOST: postgres
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: secret
-    volumes:
-      - ../bin:/blobber/bin
-      - ../sql:/blobber/sql
-    command: bash /blobber/bin/postgres-entrypoint.sh
-    links:
-      - postgres:postgres
   validator:
     image: validator
     environment:
       - DOCKER= true
-    depends_on:
-      - postgres-post
-    links:
-      - postgres-post:postgres-post
     volumes:
       - ${CONFIG_PATH:-../config}:/validator/config #value after :- is default value
       - ./blobber${BLOBBER}/data:/validator/data

diff --git a/docker.local/docker-compose.yml b/docker.local/docker-compose.yml
@@ -11,28 +11,10 @@ services:
       - "543${BLOBBER}:5432"
     labels:
       zchain: "postgres"
-  postgres-post:
-    image: postgres:14
-    environment:
-      POSTGRES_PORT: 5432
-      POSTGRES_HOST: postgres
-      POSTGRES_USER: postgres
-    volumes:
-      - ../bin:/blobber/bin
-      - ../sql:/blobber/sql
-    labels:
-      zchain: "postgres-post"
-    command: bash /blobber/bin/postgres-entrypoint.sh
-    links:
-      - postgres:postgres
   validator:
     image: validator
     environment:
       - DOCKER= true
-    depends_on:
-      - postgres-post
-    links:
-      - postgres-post:postgres-post
     volumes:
       - ../config:/validator/config
       - ./blobber${BLOBBER}/data:/validator/data

diff --git a/docker.local/https.docker-compose.yml b/docker.local/https.docker-compose.yml
@@ -4,34 +4,17 @@ services:
     image: postgres:14
     volumes:
       - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data
+      - ./sql_init:/docker-entrypoint-initdb.d
     networks:
       default:
     ports:
       - "543${BLOBBER}:5432"
     labels:
       zchain: "postgres"
-  postgres-post:
-    image: postgres:14
-    environment:
-      POSTGRES_PORT: 5432
-      POSTGRES_HOST: postgres
-      POSTGRES_USER: postgres
-    volumes:
-      - ../bin:/blobber/bin
-      - ../sql:/blobber/sql
-    labels:
-      zchain: "postgres-post"
-    command: bash /blobber/bin/postgres-entrypoint.sh
-    links:
-      - postgres:postgres
   validator:
     image: validator
     environment:
       - DOCKER= true
-    depends_on:
-      - postgres-post
-    links:
-      - postgres-post:postgres-post
     volumes:
       - ../config:/validator/config
       - ./blobber${BLOBBER}/data:/validator/data