From fdde95beabc193a198f0958e653be03cb33692d4 Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 19 Dec 2023 22:21:44 +0200 Subject: [PATCH] Sprint 1.11 (#1279) * Debug (#1232) * Revert "Debug (#1232)" (#1240) This reverts commit 49a5c9ee765a54f70ca55bc8552b599ebf125747. * Fix phase locking (#1230) (#1241) * add logs for 2 phase lock * add update log * add root in log * add update log * use update lock * add log in update repo * use save * use exec * log rows affected * use no key update * use repo update * cleanup * add defer * add defer * increase timeout by 5 mins * Case 3 and case 4 (#1198) * Send WM commit status to conductor * Rename file * Modify log type * Add state for stopping blobber from committing WM * Send file meta root to rpc server * Export function to use it in conductor client * Modify field type * Add code to get file meta root * Modify file meta root retrieval * Add field * Apply rename for conductor test based on conductor state * Add comment * Log error --------- Co-authored-by: Ebrahim Gomaa * Feature/ Challenge based on rounds (#1226) * Fix * Updated gosdk * Fix * Fix * Fix * Logging * Debug * Fix * Fix * Debug * Debug * Debug * Fix unit tests * Logigng * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Fix * Fix * Updated gosdk * Fix * Fix * log config too * Fix phase locking (#1230) * add logs for 2 phase lock * add update log * add root in log * add update log * use update lock * add log in update repo * use save * use exec * log rows affected * use no key update * use repo update * cleanup * add defer * add defer * Fix * Added logging * Debug * Debug * Added reference to logs * Fix * Fix * Updated gosdk * Use slice instead of ll * Validator last 5 transactions * Fix * Fix * Updated gosdk * Resolved comments * Updated gosdk --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> * Increase limit on number of open challenges per fetching (#1249) * Fix * Updated gosdk * Fix * Fix * Fix * Logging * Debug * Fix * Fix * Debug * Debug * Debug * Fix unit tests * Logigng * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Fix * Fix * Updated gosdk * Fix * Fix * log config too * Fix phase locking (#1230) * add logs for 2 phase lock * add update log * add root in log * add update log * use update lock * add log in update repo * use save * use exec * log rows affected * use no key update * use repo update * cleanup * add defer * add defer * Fix * Added logging * Debug * Debug * Added reference to logs * Fix * Fix * Updated gosdk * Use slice instead of ll * Validator last 5 transactions * Fix * Fix * Updated gosdk * Resolved comments * Updated gosdk * Fix * Increase time limit * Increase time limit * Fix * Fix * Fix * Fix * Debug * Debug * Debug * Fix * Fix * Fix * Cleanup logging --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> * Refactor commit (#1239) * add szwg to commit * lock for query * aggregate queries * check len * rmv debug * only update file ref * move update fields to ref * fix numBlock download stats * rmv log * rmv commented collector code * Error on renaming dir (#1250) * throw err when renaming dir * fix unit test * Fix replace blobber (#1251) * Async processing (#1225) * fix pre download * rmv hash * add async processing * use range * fix unit test * fix delete change * add logs * fix unit test * fix delete cmd * save file ref * add conn timing logs * set default to 32MB * fix timing log * add inner lock * fix test * add ctx to cancel go routine * parallel write to file * fix connection * revert storage changes * empty commit * Fix phase locking (#1230) * add logs for 2 phase lock * add update log * add root in log * add update log * use update lock * add log in update repo * use save * use exec * log rows affected * use no key update * use repo update * cleanup * add defer * add defer * rmv allocObj from connObj * Download block limit (#1254) * add limit as 500 to numBlocks * add daily and req limit * fix test * fix test * Fix download stats (#1256) * fix download stats * fix typo * Fix empty alloc cleanup (#1252) * Fix replace blobber * Fix empty alloc cleanup * Debug Rounds fetch (#1255) * Fix * Updated gosdk * Fix * Fix * Fix * Logging * Debug * Fix * Fix * Debug * Debug * Debug * Fix unit tests * Logigng * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Fix * Fix * Updated gosdk * Fix * Fix * log config too * Fix phase locking (#1230) * add logs for 2 phase lock * add update log * add root in log * add update log * use update lock * add log in update repo * use save * use exec * log rows affected * use no key update * use repo update * cleanup * add defer * add defer * Fix * Added logging * Debug * Debug * Added reference to logs * Fix * Fix * Updated gosdk * Use slice instead of ll * Validator last 5 transactions * Fix * Fix * Updated gosdk * Resolved comments * Updated gosdk * Fix * Increase time limit * Increase time limit * Fix * Fix * Fix * Fix * Debug * Debug * Debug * Fix * Fix * Fix * Cleanup logging * Fix * Debug * Fix --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> * remove prefix (#1257) * remove .gpg file if already exist * introduce flock to apt commands * check on docker context existance and install git * change buildx to action * refine buildx build name based on runner name * create docker context based on runner name * increase blobber and validator job timeout * Refine docker buildx creation and usage * Create Unique buildx per job per runner * remove qemu and upgrade go action version to v4 * changed runners labels build-&-publish-docker-image.yml * added server cleanup along with blobber build * ADD disk cleanup for conductor test * fix cleanup script * fix cleanup script * added sudo to perform cleanup * bypassing cleanup failure * bypassing cleanup failure for validators * Fix Save change (#1263) * add and update change * fix empty allocation id in change * add rename dir * Fix move change when srcPath is same as destPath (#1270) * fix move change when dest is same as src * add fileID for new ref * fix unit test * Fix challenge timing issue (#1277) * Fix challenge timing issue * Fix challenge timing issue * Fix challenge timing issue * bypass cleanup if it is already running (#1280) * Revert "Fix challenge timing issue (#1277)" (#1283) * Limit Max File Size on blobber (#1274) * Added update in config * 5TB config * Reverted max_file_size config on blobber and fetching from 0chain * Fix * Fix * Fix * Fix * Fix * Fix unit tests --------- Co-authored-by: dabasov * add l2 caching for alloc (#1258) * add l2 caching for alloc * rmv return from commit repo * use update * fix update * add log * rmv update object * add log for allocation * move lock to middleware * fix save alloc update * empty commit * fix blobber size update * fix unit test * cleanup * rmv commit in initMap * fix unit test * fix renamefile mock db * add commit method to enhancedDB --------- Co-authored-by: Yury * add list query param (#1273) * Optimize GetRefs, correct and add indexes on ref table (#1284) * Correct index on ref table, add path index for get refs optimization * Fix gorm tags on ref table * Fix typo * fix goose migration tags * add pg_trgm extension * sharder keep list (#1266) * changed get round * updated version * Fix/finalize (#1285) * Debug finalize * Debug finalize * Debug finalize * Debug finalize * Debug finalize * Fix finalize * Debug * Fix * FIX * Fixed * Fixed * fix list file (#1288) * Cleanup worker for challenge tables (#1278) * add cleanup worker for challenge tables * move cleanupGap to config --------- Co-authored-by: Yury * add err check and increase wait time (#1289) * Client stats and blacklist (#1286) * add client stats and blacklist * add client stats check * fix goose migration * fix migration * fix typo --------- Co-authored-by: Yury * Fix nested rename dir (#1295) * add log for rootRef * add objTreePath * fix rename nested dir * use save (#1297) * fix retry redeem wm check (#1299) * Use rename in place of copy buffer (#1298) * use rename than copy buffer * fix storage tests * fix update latest wm (#1301) * Fix blobber stats (#1305) * fix blobber stats * add stats worker * rmv listAlloc * build stats on start * Feat/remove native sc calls (#1306) * removed native MakeSCRestAPICall * removed native MakeSCRestAPICall --------- Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> * WM cleanup (#1296) * add cleanup worker for wm * add hdd tablespace * add log for hdd path * update default hdd path * update owner * add init db script * we are already creating with tablespace in initdb.sh * update initdb script.sh * update sql for write_markers_archive table --------- Co-authored-by: Manohar Reddy Co-authored-by: Yury * Fix/max file size updateWorker and added config for storagesc update interval (#1308) * Added worker for max file size * Added config for update worker * Debug * Added logs for update worker * Added fix for worker timings = * Debug * Debug * Debug * Debug * Debug * Removed logging * Removed logging * Refactor = * Refactor * fix tablespace permission issue in conductor test b0docker-compose.yml * Fix/cond tablespace (#1309) * updated docker-compose wrt conductor test tablspace * updated docker-compose wrt conductor test tablspace * updated docker-compose wrt conductor test tablspace * updated blobber env * update gosdk version to latest of sprint-1.11 (#1310) * fix blobber stats (#1314) * race condition fix (#1316) * Node recovery conductor tests (#1259) * notify on validator tickert generatd * fix race condition * Added changes for fail upload commit (#1318) * Feat/download verify ct (#1317) * notify on validator tickert generatd * fix race condition * miss up download for CT * debug logs * Improve download performance (#1315) * improve download file * write data * rmv content length * fix unit test * add new db snapshot migration (#1312) Co-authored-by: Yury * fix commit error (#1320) * update gosdk (#1319) Co-authored-by: Yaroslav Svitlytskyi <53532703+YarikRevich@users.noreply.github.com> * fix where statement (#1321) * Fix/loop break (#1322) * updated gosdk * add mutex * updated gosdk * merged changes * merged changes --------- Co-authored-by: Hitenjain14 * Fix blobber stats panic (#1324) * Fix blobber stats panic * Fix * Fix * fix break in switch (#1326) * change hasher to blake3 (#1325) * change hasher to blake3 * add blake hash * fix lint * fix lint for hash write * update gosdk * Fix challenge worker (#1327) * revert changes * fix challenge worker * fix lint * add goto for accept case --------- Co-authored-by: Yury * Config watcher (#1329) * add watch config * add on config change * Remove min lock demand from blobber (#1332) * fix alloc lock (#1334) * Feature: implement fork per run strategy (#1335) * feature: added fork creation * feature: improved Tenderly fork creation flow * Merge pull request #1336 * rmv wm lock table * fix unit tests * rmv write_locks table * rmv gorm tag * empty commit * Fix mem usage (#1337) * updated gosdk to blobber * change hasher to sha2 256 * fix unit test * changed runner for lint tests.yml --------- Co-authored-by: shahnawaz-creator Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> * fix rb for failed marker (#1342) * Fix lwm (#1338) * add wm log * fix log * fix update allocation * fix log * lru of value * fix alloc update * Fix auth ticket (#1341) * fix read auth ticket * fix unit test * convert to base64 * Fix expiration column name (#1343) * Fix expiration column name * Fix size param * merged changes * Revert "merged changes" This reverts commit 39f717ab27388171bd3198a0f8ef945b110482a9. * feature: added custom block number during Tenderly fork creation (#1345) * fix update alloc (#1347) * extended field for mimetype * add aws secrets for blobbers (#1349) * add aws secrets for blobbers * add validator secrets * typo * update logs --------- Co-authored-by: sanchit * Fix root hash (#1346) * add root logs * check alloc root * fix lint * rmv wm cleanup * fix lint * fix encrypted key point (#1350) * fix encrypted key point * check root ref precommit * update min-submit to 20% (#1348) * update gosdk to v1.11.0 (#1351) * fix wm retries (#1352) --------- Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Kishan Dhakan Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Laxmi Prasad Oli Co-authored-by: Ebrahim Gomaa Co-authored-by: Amr Amin Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: shahnawaz-creator Co-authored-by: tapishsinha-rs Co-authored-by: Dinmukhammed Kambarov <52813950+din-mukhammed@users.noreply.github.com> Co-authored-by: Manohar Reddy Co-authored-by: Yaroslav Svitlytskyi <53532703+YarikRevich@users.noreply.github.com> Co-authored-by: Hitenjain14 Co-authored-by: Sanchit Sharma <47826073+Sanchit011@users.noreply.github.com> --- .../build-&-publish-docker-image.yml | 103 ++++- .github/workflows/system_tests.yml | 3 +- .github/workflows/tests.yml | 2 +- README.md | 4 - code/go/0chain.net/blobber/config.go | 72 +--- code/go/0chain.net/blobber/datastore.go | 3 +- code/go/0chain.net/blobber/http.go | 8 +- code/go/0chain.net/blobber/main.go | 4 +- code/go/0chain.net/blobber/node.go | 45 ++- code/go/0chain.net/blobber/settings.go | 54 ++- code/go/0chain.net/blobber/worker.go | 7 +- code/go/0chain.net/blobber/zcn.go | 6 + .../allocation/allocationchange.go | 61 ++- .../blobbercore/allocation/connection.go | 360 ++++++++++++++++-- .../blobbercore/allocation/copyfilechange.go | 3 +- .../allocation/copyfilechange_test.go | 5 +- .../allocation/deletefilechange.go | 5 +- .../allocation/deletefilechange_test.go | 5 +- .../blobbercore/allocation/file_changer.go | 3 +- .../allocation/file_changer_base.go | 46 ++- .../allocation/file_changer_update.go | 9 +- .../allocation/file_changer_upload.go | 3 +- .../file_changer_upload_integration.go | 28 ++ .../allocation/file_changer_upload_main.go | 15 + .../allocation/file_changer_upload_test.go | 6 +- .../blobbercore/allocation/movefilechange.go | 83 ++-- .../allocation/movefilechange_test.go | 17 +- .../blobbercore/allocation/newdirchange.go | 3 +- .../blobbercore/allocation/protocol.go | 36 +- .../allocation/renamefilechange.go | 10 +- .../renamefilechange_integration.go | 28 ++ .../allocation/renamefilechange_main.go | 17 + .../allocation/renamefilechange_test.go | 2 +- .../blobbercore/allocation/repository.go | 209 +++++++++- .../allocation/updatefilechange_test.go | 6 +- .../blobbercore/allocation/workers.go | 124 +++++- .../0chain.net/blobbercore/allocation/zcn.go | 2 +- .../blobbercore/blobberhttp/response.go | 24 +- .../blobbercore/challenge/challenge.go | 41 +- .../blobbercore/challenge/entity.go | 29 ++ .../blobbercore/challenge/protocol.go | 68 ++-- .../blobbercore/challenge/timing.go | 22 ++ .../blobbercore/challenge/worker.go | 62 ++- .../0chain.net/blobbercore/config/config.go | 112 +++++- .../blobbercore/convert/response_creator.go | 6 +- .../blobbercore/convert/response_handler.go | 4 +- .../blobbercore/datastore/postgres.go | 10 +- .../0chain.net/blobbercore/datastore/store.go | 15 +- .../0chain.net/blobbercore/filestore/state.go | 1 - .../blobbercore/filestore/storage.go | 48 +-- .../0chain.net/blobbercore/filestore/store.go | 1 + .../blobbercore/filestore/store_test.go | 21 +- .../blobbercore/filestore/tree_validation.go | 16 +- .../filestore/tree_validation_bench_test.go | 6 +- .../tree_validation_integration_tests.go | 59 +++ .../filestore/tree_validation_main.go | 11 + .../filestore/tree_validation_test.go | 4 +- .../blobbercore/handler/client_quota.go | 153 ++++++++ .../blobbercore/handler/download_quota.go | 35 +- .../blobbercore/handler/file_command.go | 22 +- .../handler/file_command_delete.go | 42 +- .../handler/file_command_update.go | 158 ++++---- .../handler/file_command_upload.go | 150 ++++---- .../blobbercore/handler/grpc_handler_test.go | 1 + .../0chain.net/blobbercore/handler/handler.go | 31 +- .../blobbercore/handler/handler_common.go | 21 +- .../handler/handler_download_test.go | 26 +- .../handler/handler_objecttree_test.go | 2 +- .../blobbercore/handler/handler_share_test.go | 2 +- .../blobbercore/handler/handler_test.go | 12 +- .../handler/object_operation_handler.go | 251 ++++++------ .../handler/object_operation_handler_test.go | 10 +- .../blobbercore/handler/storage_handler.go | 39 +- .../0chain.net/blobbercore/handler/worker.go | 2 + .../blobbercore/readmarker/protocol.go | 4 +- .../blobbercore/readmarker/worker.go | 4 +- .../blobbercore/reference/dbCollector.go | 49 +++ .../blobbercore/reference/filestats.go | 59 ++- .../0chain.net/blobbercore/reference/ref.go | 96 ++--- .../blobbercore/stats/blobberstats.go | 106 ++---- .../0chain.net/blobbercore/stats/handler.go | 26 +- .../blobbercore/writemarker/entity.go | 68 ++-- .../blobbercore/writemarker/mutex.go | 71 ++-- .../blobbercore/writemarker/mutext_test.go | 57 +-- .../blobbercore/writemarker/protocol.go | 11 +- .../blobbercore/writemarker/worker.go | 45 ++- .../blobbercore/writemarker/write_lock.go | 14 +- .../0chain.net/conductor/conductrpc/client.go | 10 + .../0chain.net/conductor/conductrpc/entity.go | 24 ++ .../conductor/conductrpc/file_meta.go | 64 ++++ .../0chain.net/conductor/conductrpc/server.go | 4 + .../0chain.net/conductor/conductrpc/state.go | 5 + code/go/0chain.net/core/common/admin.go | 26 +- code/go/0chain.net/core/common/aws.go | 32 ++ code/go/0chain.net/core/common/constants.go | 3 + code/go/0chain.net/core/common/handler.go | 7 +- .../go/0chain.net/core/common/request_form.go | 2 +- code/go/0chain.net/core/encryption/hash.go | 18 + code/go/0chain.net/core/lock/lock.go | 3 + code/go/0chain.net/core/transaction/entity.go | 33 +- code/go/0chain.net/core/transaction/http.go | 139 +------ code/go/0chain.net/swagger.md | 1 - code/go/0chain.net/validator/main.go | 45 ++- .../challenge_handler_integration_tests.go | 6 + .../storage/challenge_handler_main.go | 9 +- .../validatorcore/storage/handler_main.go | 2 +- .../validatorcore/storage/models.go | 6 +- .../validatorcore/storage/protocol.go | 2 +- .../validatorcore/storage/stats_handler.go | 24 +- config/0chain_blobber.yaml | 31 +- config/0chain_validator.yaml | 6 +- docker.local/b0docker-compose.yml | 41 +- .../conductor-config/0chain_blobber.yaml | 5 +- .../conductor-config/0chain_validator.yaml | 2 +- docker.local/sql_init/000-init-db.sh | 8 + go.mod | 39 +- go.sum | 64 +++- ...ta.sql => 1698861371_full_db_snapshot.sql} | 314 ++++++++------- 118 files changed, 3090 insertions(+), 1409 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go create mode 100644 code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go create mode 100644 code/go/0chain.net/blobbercore/allocation/renamefilechange_integration.go create mode 100644 code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go create mode 100644 code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go create mode 100644 code/go/0chain.net/blobbercore/filestore/tree_validation_main.go create mode 100644 code/go/0chain.net/blobbercore/handler/client_quota.go create mode 100644 code/go/0chain.net/blobbercore/reference/dbCollector.go create mode 100644 code/go/0chain.net/conductor/conductrpc/file_meta.go create mode 100644 code/go/0chain.net/core/common/aws.go create mode 100755 docker.local/sql_init/000-init-db.sh rename goose/migrations/{001_blobber_meta.sql => 1698861371_full_db_snapshot.sql} (68%) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 6dee115c7..6f7b8901e 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Cleanup before restarting conductor tests. run: | - echo 'y' | docker system prune -a + echo 'y' | docker system prune -a || true cd /tmp sudo rm -rf ./* @@ -99,7 +99,7 @@ jobs: steps: - name: Cleanup before restarting conductor tests. run: | - echo 'y' | docker system prune -a + echo 'y' | docker system prune -a || true cd /tmp sudo rm -rf ./* @@ -204,6 +204,90 @@ jobs: echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV + - name: 'Setup jq' + uses: dcarbone/install-jq-action@v2.1.0 + with: + version: '1.7' + force: 'false' + + - name: "Create Tenderly fork" + run: | + echo "TENDERLY_CREATION_INFO=$(curl -X POST \ + -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \ + -H "Content-Type: application/json" \ + -d '{"alias":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}${{ github.run_attempt }}", "description":"", "block_number": 18539779, "network_id":"1"}' \ + https://api.tenderly.co/api/v1/account/zus_network/project/project/fork)" >> $GITHUB_ENV + + - name: "Parse Tenderly fork creation transaction result" + run: | + echo "TENDERLY_FORK_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.simulation_fork.id')" >> $GITHUB_ENV + echo "TENDERLY_ROOT_TRANSACTION_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.root_transaction.id')" >> $GITHUB_ENV + + - name: "Retrieve Tenderly fork block number" + run: | + echo "TENDERLY_FORK_BLOCK_NUMBER=$(curl -X GET \ + -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \ + -H "Content-Type: application/json" \ + https://api.tenderly.co/api/v1/network/1/block-number | jq -r '.block_number')" >> $GITHUB_ENV + + echo "TENDERLY_FORK_BLOCK_NUMBER=$((${{ env.TENDERLY_FORK_BLOCK_NUMBER }} + 1))" >> GITHUB_ENV + + - name: "Transfer Bridge ownership in Tenderly fork" + run: | + echo "TENDERLY_ROOT_TRANSACTION_ID=$(curl -X POST \ + -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \ + -H "Content-Type: application/json" \ + -d '{ + "network_id": "1", + "block_number": ${{ env.TENDERLY_FORK_BLOCK_NUMBER }}, + "transaction_index": null, + "from": "0xed8f3170db6d1a71c8fa6d8d73cc2c51db95d5a4", + "input": "0xf2fde38b0000000000000000000000008e25cfd9bd6c0ca67a5522cd920b3c66d39d6e97", + "to": "0x7700d773022b19622095118fadf46f7b9448be9b", + "gas": 8000000, + "gas_price": "0", + "value": "0", + "access_list": [], + "generate_access_list": true, + "save": true, + "source": "dashboard", + "block_header": null, + "root": "${{ env.TENDERLY_ROOT_TRANSACTION_ID }}", + "skip_fork_head_update": false, + "alias": "", + "description": "Transfer ownership to 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97" + }' \ + https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }}/simulate | jq -r '.simulation.id')" >> $GITHUB_ENV + + echo "TENDERLY_FORK_BLOCK_NUMBER=$((${{ env.TENDERLY_FORK_BLOCK_NUMBER }} + 1))" >> GITHUB_ENV + + - name: "Transfer Authorizers ownership in Tenderly fork" + run: | + curl -X POST \ + -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \ + -H "Content-Type: application/json" \ + -d '{ + "network_id": "1", + "block_number": ${{ env.TENDERLY_FORK_BLOCK_NUMBER }}, + "transaction_index": null, + "from": "0xed8f3170db6d1a71c8fa6d8d73cc2c51db95d5a4", + "input": "0xf2fde38b0000000000000000000000008e25cfd9bd6c0ca67a5522cd920b3c66d39d6e97", + "to": "0x481dab4407b9880de0a68dc62e6af611c4949e42", + "gas": 8000000, + "gas_price": "0", + "value": "0", + "access_list": [], + "generate_access_list": true, + "save": true, + "source": "dashboard", + "block_header": null, + "root": "${{ env.TENDERLY_ROOT_TRANSACTION_ID }}", + "skip_fork_head_update": false, + "alias": "", + "description": "Transfer ownership to 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97" + }' \ + https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }}/simulate + - name: "Deploy 0Chain" uses: 0chain/actions/deploy-0chain@master with: @@ -213,13 +297,12 @@ jobs: blobber_image: ${{ env.TAG }}-${{ env.SHORT_SHA }} validator_image: ${{ env.TAG }}-${{ env.SHORT_SHA }} SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }} - TENDERLY_FORK_ID: ${{ secrets.TENDERLY_FORK_ID }} + TENDERLY_FORK_ID: ${{ env.TENDERLY_FORK_ID }} graphnode_sc: ${{ secrets.GRAPHNODE_SC }} graphnode_network: ${{ secrets.GRAPHNODE_NETWORK }} - graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/${{ secrets.TENDERLY_FORK_ID }} + graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/${{ env.TENDERLY_FORK_ID }} svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} - - name: "Run System tests" uses: 0chain/actions/run-system-tests@master with: @@ -231,11 +314,19 @@ jobs: run_flaky_tests: false retry_failures: true run_smoke_tests: ${{ github.ref != 'refs/heads/staging' && github.base_ref != 'staging' && github.ref != 'refs/heads/master' && github.base_ref != 'master' }} - TENDERLY_FORK_ID: ${{ secrets.TENDERLY_FORK_ID }} + TENDERLY_FORK_ID: ${{ env.TENDERLY_FORK_ID }} DEVOPS_CHANNEL_WEBHOOK_URL: ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} + - name: "Remove Tenderly fork" + if: always() + run: | + curl -X DELETE \ + -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \ + -H "Content-Type: application/json" \ + https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }} + - name: "Set PR status as ${{ job.status }}" if: ${{ (success() || failure()) && steps.findPr.outputs.number }} uses: 0chain/actions/set-pr-status@master diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 88e1b6d68..53138d0ad 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -46,9 +46,8 @@ jobs: DEV8KC: ${{ secrets.DEV8KC }} DEV9KC: ${{ secrets.DEV9KC }} SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }} - TENDERLY_FORK_ID: ${{ secrets.TENDERLY_FORK_ID }} + TENDERLY_SECRET: ${{ secrets.TENDERLY_SECRET }} GRAPHNODE_NETWORK: ${{ secrets.GRAPHNODE_NETWORK }} - GRAPHNODE_ETHEREUM_NODE_URL: https://rpc.tenderly.co/fork/${{ secrets.TENDERLY_FORK_ID }} DEVOPS_CHANNEL_WEBHOOK_URL: ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }} GRAPHNODE_SC: ${{ secrets.GRAPHNODE_SC }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7991a985c..2bf1fe9db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: linter: name: Lints - runs-on: [self-hosted,arc-runner] + runs-on: [self-hosted,blobber-runner] steps: - name: Setup go uses: actions/setup-go@v3 diff --git a/README.md b/README.md index a7d5ac0c5..fdf60581e 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,6 @@ Sample Response: terms: read_price: 26.874 mZCN / GB write_price: 26.874 mZCN / GB / time_unit - min_lock_demand: 0.1 cct: 2m0s max_offer_duration: 744h0m0s - id: 7a90e6790bcd3d78422d7a230390edc102870fe58c15472073922024985b1c7d @@ -176,7 +175,6 @@ Sample Response: terms: read_price: 10.000 mZCN / GB write_price: 100.000 mZCN / GB / time_unit - min_lock_demand: 0.1 cct: 2m0s max_offer_duration: 744h0m0s - id: f65af5d64000c7cd2883f4910eb69086f9d6e6635c744e62afcfab58b938ee25 @@ -186,7 +184,6 @@ Sample Response: terms: read_price: 10.000 mZCN / GB write_price: 100.000 mZCN / GB / time_unit - min_lock_demand: 0.1 cct: 2m0s max_offer_duration: 744h0m0s - id: f8dc4aaf3bb32ae0f4ed575dd6931a42b75e546e07cb37a6e1c6aaf1225891c5 @@ -196,7 +193,6 @@ Sample Response: terms: read_price: 26.874 mZCN / GB write_price: 26.865 mZCN / GB / time_unit - min_lock_demand: 0.1 cct: 2m0s max_offer_duration: 744h0m0s ``` diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 55a80b794..80979e41e 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -3,8 +3,6 @@ package main import ( "context" "fmt" - "log" - "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" @@ -29,76 +27,8 @@ func setupConfig(configDir string, deploymentMode int) { if config.Configuration.MountPoint == "" { panic("Please specify mount point in flag or config file") } - config.Configuration.AllocDirLevel = viper.GetIntSlice("storage.alloc_dir_level") - config.Configuration.FileDirLevel = viper.GetIntSlice("storage.file_dir_level") - config.Configuration.DeploymentMode = byte(deploymentMode) - config.Configuration.ChainID = viper.GetString("server_chain.id") - config.Configuration.SignatureScheme = viper.GetString("server_chain.signature_scheme") - - config.Configuration.OpenConnectionWorkerFreq = viper.GetInt64("openconnection_cleaner.frequency") - config.Configuration.OpenConnectionWorkerTolerance = viper.GetInt64("openconnection_cleaner.tolerance") - - config.Configuration.WMRedeemFreq = viper.GetInt64("writemarker_redeem.frequency") - config.Configuration.WMRedeemNumWorkers = viper.GetInt("writemarker_redeem.num_workers") - - config.Configuration.RMRedeemFreq = viper.GetInt64("readmarker_redeem.frequency") - config.Configuration.RMRedeemNumWorkers = viper.GetInt("readmarker_redeem.num_workers") - - config.Configuration.HealthCheckWorkerFreq = viper.GetDuration("healthcheck.frequency") - - config.Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency") - config.Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers") - config.Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries") - - config.Configuration.AutomaticUpdate = viper.GetBool("disk_update.automatic_update") - blobberUpdateIntrv := viper.GetDuration("disk_update.blobber_update_interval") - if blobberUpdateIntrv <= 0 { - blobberUpdateIntrv = 5 * time.Minute - } - config.Configuration.BlobberUpdateInterval = blobberUpdateIntrv - - config.Configuration.PGUserName = viper.GetString("pg.user") - config.Configuration.PGPassword = viper.GetString("pg.password") - config.Configuration.DBHost = viper.GetString("db.host") - config.Configuration.DBName = viper.GetString("db.name") - config.Configuration.DBPort = viper.GetString("db.port") - config.Configuration.DBUserName = viper.GetString("db.user") - config.Configuration.DBPassword = viper.GetString("db.password") - config.Configuration.DBTablesToKeep = viper.GetStringSlice("db.keep_tables") - - config.Configuration.PriceInUSD = viper.GetBool("price_in_usd") - - config.Configuration.WriteMarkerLockTimeout = viper.GetDuration("write_marker_lock_timeout") - - config.Configuration.UpdateAllocationsInterval = - viper.GetDuration("update_allocations_interval") - - config.Configuration.MaxAllocationDirFiles = - viper.GetInt("max_dirs_files") - if config.Configuration.MaxAllocationDirFiles < 50000 { - config.Configuration.MaxAllocationDirFiles = 50000 - } - - config.Configuration.DelegateWallet = viper.GetString("delegate_wallet") - if w := config.Configuration.DelegateWallet; len(w) != 64 { - log.Fatal("invalid delegate wallet:", w) - } - - config.Configuration.MinSubmit = viper.GetInt("min_submit") - if config.Configuration.MinSubmit < 1 { - config.Configuration.MinSubmit = 50 - } else if config.Configuration.MinSubmit > 100 { - config.Configuration.MinSubmit = 100 - } - config.Configuration.MinConfirmation = viper.GetInt("min_confirmation") - if config.Configuration.MinConfirmation < 1 { - config.Configuration.MinConfirmation = 50 - } else if config.Configuration.MinConfirmation > 100 { - config.Configuration.MinConfirmation = 100 - } - transaction.MinConfirmation = config.Configuration.MinConfirmation - + config.ReadConfig(deploymentMode) fmt.Print(" [OK]\n") } diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index 4a168df7f..26b39e143 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -34,7 +34,6 @@ func setupDatabase() error { time.Sleep(1 * time.Second) } - if err := migrateDatabase(pgDB); err != nil { return fmt.Errorf("error while migrating schema: %v", err) } @@ -49,7 +48,7 @@ func migrateDatabase(db *gorm.DB) error { if err != nil { return err } - + goose.Migrate(sqlDB) return nil } diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 0ec89c658..9846c27f4 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -26,7 +26,7 @@ func startHttpServer() { } r := mux.NewRouter() - initHandlers(r) + initHandlers(r, config.Development()) var wg sync.WaitGroup @@ -36,7 +36,7 @@ func startHttpServer() { // start https server go startServer(&wg, r, mode, httpsPort, true) - logging.Logger.Info("Ready to listen to the requests") + logging.Logger.Info("Ready to listen to the requests with development mode: " + mode) fmt.Print("> start http server [OK]\n") wg.Wait() @@ -104,12 +104,12 @@ func startServer(wg *sync.WaitGroup, r *mux.Router, mode string, port int, isTls } } -func initHandlers(r *mux.Router) { +func initHandlers(r *mux.Router, devMode bool) { handler.StartTime = time.Now().UTC() r.HandleFunc("/", handler.HomepageHandler) handler.SetupHandlers(r) handler.SetupSwagger() - common.SetAdminCredentials() + common.SetAdminCredentials(devMode) } func initProfHandlers(mux *http.ServeMux) { diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 7ee5a3202..06db9d222 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -48,8 +48,8 @@ func main() { panic(err) } - err := setCCTFromChain() - if err != nil { + if err := setStorageScConfigFromChain(); err != nil { + logging.Logger.Error("Error setStorageScConfigFromChain" + err.Error()) panic(err) } diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 1da96a289..6b1a2b06b 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -4,23 +4,30 @@ import ( "errors" "fmt" "os" + "strings" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "go.uber.org/zap" ) +var publicKey, privateKey string + func setupNode() error { - fmt.Print("> setup blobber") + fmt.Println("> setup blobber") - reader, err := os.Open(keysFile) + err := readKeysFromAws() if err != nil { - return err + err = readKeysFromFile(&keysFile) + if err != nil { + panic(err) + } + fmt.Println("using blobber keys from local") + } else { + fmt.Println("using blobber keys from aws") } - defer reader.Close() - - publicKey, privateKey, _, _ := encryption.ReadKeys(reader) node.Self.SetKeys(publicKey, privateKey) if node.Self.ID == "" { @@ -48,3 +55,29 @@ func setupNode() error { fmt.Print(" [OK]\n") return nil } + +func readKeysFromAws() error { + blobberSecretName := os.Getenv("BLOBBER_SECRET_NAME") + awsRegion := os.Getenv("AWS_REGION") + keys, err := common.GetSecretsFromAWS(blobberSecretName, awsRegion) + if err != nil { + return err + } + secretsFromAws := strings.Split(keys, "\n") + if len(secretsFromAws) < 2 { + return fmt.Errorf("wrong file format from aws") + } + publicKey = secretsFromAws[0] + privateKey = secretsFromAws[1] + return nil +} + +func readKeysFromFile(keysFile *string) error { + reader, err := os.Open(*keysFile) + if err != nil { + return err + } + defer reader.Close() + publicKey, privateKey, _, _ = encryption.ReadKeys(reader) + return nil +} \ No newline at end of file diff --git a/code/go/0chain.net/blobber/settings.go b/code/go/0chain.net/blobber/settings.go index c6dcd88eb..eb90322b2 100644 --- a/code/go/0chain.net/blobber/settings.go +++ b/code/go/0chain.net/blobber/settings.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strconv" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -11,42 +12,53 @@ import ( "github.com/0chain/gosdk/zcncore" ) -type cctCB struct { +type storageScCB struct { done chan struct{} - cct time.Duration + cct int64 + mfs int64 err error } -func (c *cctCB) OnInfoAvailable(op int, status int, info string, errStr string) { +func (ssc *storageScCB) OnInfoAvailable(op int, status int, info string, errStr string) { defer func() { - c.done <- struct{}{} + ssc.done <- struct{}{} }() if errStr != "" { - c.err = errors.New(errStr) + ssc.err = errors.New(errStr) return } m := make(map[string]interface{}) err := json.Unmarshal([]byte(info), &m) if err != nil { - c.err = err + ssc.err = err return } m = m["fields"].(map[string]interface{}) - cct := m["max_challenge_completion_time"].(string) - d, err := time.ParseDuration(cct) + cctString := m["max_challenge_completion_rounds"].(string) + mfsString := m["max_file_size"].(string) + + cct, err := strconv.ParseInt(cctString, 10, 64) + if err != nil { + ssc.err = err + return + } + + mfs, err := strconv.ParseInt(mfsString, 10, 64) if err != nil { - c.err = err + ssc.err = err return } - c.cct = d + + ssc.cct = cct + ssc.mfs = mfs } -func setCCTFromChain() error { - cb := &cctCB{ +func setStorageScConfigFromChain() error { + cb := &storageScCB{ done: make(chan struct{}), } err := zcncore.GetStorageSCConfig(cb) @@ -59,24 +71,24 @@ func setCCTFromChain() error { } config.StorageSCConfig.ChallengeCompletionTime = cb.cct + config.StorageSCConfig.MaxFileSize = cb.mfs return nil } -func updateCCTWorker(ctx context.Context) { - ticker := time.NewTicker(time.Hour) +func updateStorageScConfigWorker(ctx context.Context) { + interval := time.Hour + if config.Development() { + interval = time.Second + } + + ticker := time.NewTicker(interval) for { select { case <-ctx.Done(): return case <-ticker.C: - // We'd panic if err occurred when calling setCCTFromChain from - // main.go file because cct would be initially 0 and we cannot - // work with 0 value. - // Upon updating cct, we only log error because cct is not 0 - // We should try to submit challenge as soon as possible regardless - // of cct value. - err := setCCTFromChain() + err := setStorageScConfigFromChain() if err != nil { logging.Logger.Error(err.Error()) } diff --git a/code/go/0chain.net/blobber/worker.go b/code/go/0chain.net/blobber/worker.go index a5d5aa5c9..92db82a86 100644 --- a/code/go/0chain.net/blobber/worker.go +++ b/code/go/0chain.net/blobber/worker.go @@ -10,6 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -22,8 +23,12 @@ func setupWorkers(ctx context.Context) { readmarker.SetupWorkers(ctx) writemarker.SetupWorkers(ctx) allocation.StartUpdateWorker(ctx, config.Configuration.UpdateAllocationsInterval) + allocation.StartFinalizeWorker(ctx, config.Configuration.FinalizeAllocationsInterval) allocation.SetupWorkers(ctx) - updateCCTWorker(ctx) + challenge.SetupChallengeCleanUpWorker(ctx) + challenge.SetupChallengeTimingsCleanupWorker(ctx) + stats.SetupStatsWorker(ctx) + updateStorageScConfigWorker(ctx) } // startRefreshSettings sync settings from blockchain diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index afec19c90..f59e599cc 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -11,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" handleCommon "github.com/0chain/blobber/code/go/0chain.net/core/common/handler" "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/gosdk/zboxcore/sdk" "github.com/0chain/gosdk/zcncore" ) @@ -83,6 +84,11 @@ func setupServerChain() error { return err } + if err := sdk.InitStorageSDK(node.Self.GetWalletString(), serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, + nil, 0); err != nil { + return err + } + fmt.Print(" [OK]\n") return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 7549a24d4..e31c3bef8 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -14,6 +14,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" + "github.com/remeh/sizedwaitgroup" "go.uber.org/zap" "gorm.io/gorm" @@ -29,7 +30,7 @@ const ( // AllocationChangeProcessor request transaction of file operation. it is president in postgres, and can be rebuilt for next http reqeust(eg CommitHandler) type AllocationChangeProcessor interface { - CommitToFileStore(ctx context.Context) error + CommitToFileStore(ctx context.Context, mut *sync.Mutex) error DeleteTempFile() error ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) @@ -140,6 +141,7 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien cc.ComputeProperties() // Load connection Obj size from memory cc.Size = GetConnectionObjSize(connectionID) + cc.Status = InProgressConnection return cc, nil } @@ -170,6 +172,12 @@ func (cc *AllocationChangeCollector) Save(ctx context.Context) error { return db.Save(cc).Error } +func (cc *AllocationChangeCollector) Create(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + cc.Status = NewConnection + return db.Create(cc).Error +} + // ComputeProperties unmarshal all ChangeProcesses from postgres func (cc *AllocationChangeCollector) ComputeProperties() { cc.AllocationChanges = make([]AllocationChangeProcessor, 0, len(cc.Changes)) @@ -203,32 +211,57 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot string, - ts common.Timestamp, fileIDMeta map[string]string) error { + ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { rootRef, err := cc.GetRootRef(ctx) - logging.Logger.Info("GetRootRef", zap.Any("rootRef", rootRef)) if err != nil { - return err + return rootRef, err } for idx, change := range cc.Changes { changeProcessor := cc.AllocationChanges[idx] _, err := changeProcessor.ApplyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) if err != nil { - return err + return rootRef, err } } - _, err = rootRef.CalculateHash(ctx, true) - return err + collector := reference.NewCollector(len(cc.Changes)) + _, err = rootRef.CalculateHash(ctx, true, collector) + if err != nil { + return rootRef, err + } + err = collector.Finalize(ctx) + return rootRef, err } func (a *AllocationChangeCollector) CommitToFileStore(ctx context.Context) error { - + commitCtx, cancel := context.WithCancel(ctx) + defer cancel() + // Can be configured at runtime, this number will depend on the number of active allocations + swg := sizedwaitgroup.New(5) + mut := &sync.Mutex{} + var ( + commitError error + errorMutex sync.Mutex + ) for _, change := range a.AllocationChanges { - err := change.CommitToFileStore(ctx) - if err != nil { - return err + select { + case <-commitCtx.Done(): + return fmt.Errorf("commit to filestore failed: %s", commitError.Error()) + default: } + swg.Add() + go func(change AllocationChangeProcessor) { + err := change.CommitToFileStore(ctx, mut) + if err != nil && !errors.Is(common.ErrFileWasDeleted, err) { + cancel() + errorMutex.Lock() + commitError = err + errorMutex.Unlock() + } + swg.Done() + }(change) } - return nil + swg.Wait() + return commitError } func (a *AllocationChangeCollector) DeleteChanges(ctx context.Context) { @@ -248,7 +281,7 @@ type Result struct { } // TODO: Need to speed up this function -func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) (err error) { +func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { logging.Logger.Info("Move to filestore", zap.String("allocation_id", a.AllocationID)) @@ -258,7 +291,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) (err er e := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { tx := datastore.GetStore().GetTransaction(ctx) - err = tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). + err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { for _, ref := range refs { diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index ae366b768..25445f6b8 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -5,85 +5,378 @@ import ( "sync" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) var ( // ConnectionObjCleanInterval start to clean the connectionObjMap ConnectionObjCleanInterval = 10 * time.Minute // ConnectionObjTimout after which connectionObj entry should be invalid - ConnectionObjTimeout = 10 * time.Minute + ConnectionObjTimeout = 20 * time.Minute ) var ( - connectionObjSizeMap = make(map[string]*ConnectionObjSize) - connectionObjMutex sync.RWMutex + connectionProcessor = make(map[string]*ConnectionProcessor) + connectionObjMutex sync.RWMutex ) -type ConnectionObjSize struct { +type ConnectionProcessor struct { Size int64 UpdatedAt time.Time - Changes map[string]*ConnectionChanges + lock sync.RWMutex + changes map[string]*ConnectionChange + ClientID string + ctx context.Context + ctxCancel context.CancelFunc } -type ConnectionChanges struct { - Hasher *filestore.CommitHasher +type ConnectionChange struct { + hasher *filestore.CommitHasher + baseChanger *BaseFileChanger + existingRef *reference.Ref + processError error + ProcessChan chan FileCommand + wg sync.WaitGroup + isFinalized bool +} + +func CreateConnectionChange(connectionID, pathHash string, allocationObj *Allocation) *ConnectionChange { + connectionObjMutex.Lock() + connectionObj := connectionProcessor[connectionID] + if connectionObj == nil { + ctx, cancel := context.WithCancel(context.Background()) + connectionObj = &ConnectionProcessor{ + UpdatedAt: time.Now(), + changes: make(map[string]*ConnectionChange), + ctx: ctx, + ctxCancel: cancel, + } + connectionProcessor[connectionID] = connectionObj + } + connectionObjMutex.Unlock() + connectionObj.lock.Lock() + connChange := &ConnectionChange{ + ProcessChan: make(chan FileCommand, 2), + wg: sync.WaitGroup{}, + } + connectionObj.changes[pathHash] = connChange + connectionObj.lock.Unlock() + connChange.wg.Add(1) + go func() { + processCommand(connectionObj.ctx, connChange.ProcessChan, allocationObj, connectionID, connectionObj.ClientID, pathHash) + connChange.wg.Done() + }() + return connChange +} + +func GetConnectionChange(connectionID, pathHash string) *ConnectionChange { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return nil + } + return connectionObj.changes[pathHash] +} + +func GetFileChanger(connectionID, pathHash string) *BaseFileChanger { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return nil + } + connectionObj.lock.RLock() + defer connectionObj.lock.RUnlock() + if connectionObj.changes[pathHash] == nil { + return nil + } + return connectionObj.changes[pathHash].baseChanger +} + +func SaveFileChanger(connectionID string, fileChanger *BaseFileChanger) error { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return common.NewError("connection_not_found", "connection not found") + } + connectionObj.lock.Lock() + if connectionObj.changes[fileChanger.PathHash] == nil { + return common.NewError("connection_change_not_found", "connection change not found") + } + connectionObj.changes[fileChanger.PathHash].baseChanger = fileChanger + connectionObj.lock.Unlock() + return nil +} + +func SaveExistingRef(connectionID, pathHash string, existingRef *reference.Ref) error { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return common.NewError("connection_not_found", "connection not found") + } + connectionObj.lock.Lock() + defer connectionObj.lock.Unlock() + if connectionObj.changes[pathHash] == nil { + return common.NewError("connection_change_not_found", "connection change not found") + } + connectionObj.changes[pathHash].existingRef = existingRef + return nil +} + +func GetExistingRef(connectionID, pathHash string) *reference.Ref { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return nil + } + connectionObj.lock.RLock() + defer connectionObj.lock.RUnlock() + if connectionObj.changes[pathHash] == nil { + return nil + } + return connectionObj.changes[pathHash].existingRef +} + +func SetFinalized(connectionID, pathHash string, cmd FileCommand) error { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return common.NewError("connection_not_found", "connection not found") + } + connectionObj.lock.Lock() + connChange := connectionObj.changes[pathHash] + // Can happen due to resume or redundant call + if connChange.isFinalized { + connectionObj.lock.Unlock() + connChange.wg.Wait() + return nil + } + connChange.isFinalized = true + connectionObj.lock.Unlock() + connChange.ProcessChan <- cmd + close(connChange.ProcessChan) + connChange.wg.Wait() + return GetError(connectionID, pathHash) +} + +func SendCommand(connectionID, pathHash string, cmd FileCommand) error { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return common.NewError("connection_not_found", "connection not found") + } + connectionObj.lock.RLock() + defer connectionObj.lock.RUnlock() + connChange := connectionObj.changes[pathHash] + if connChange == nil { + return common.NewError("connection_change_not_found", "connection change not found") + } + if connChange.processError != nil { + return connChange.processError + } + if connChange.isFinalized { + return common.NewError("connection_change_finalized", "connection change finalized") + } + connChange.ProcessChan <- cmd + return nil +} + +func GetConnectionProcessor(connectionID string) *ConnectionProcessor { + connectionObjMutex.RLock() + defer connectionObjMutex.RUnlock() + return connectionProcessor[connectionID] +} + +func CreateConnectionProcessor(connectionID string) *ConnectionProcessor { + connectionObjMutex.Lock() + defer connectionObjMutex.Unlock() + connectionObj := connectionProcessor[connectionID] + if connectionObj == nil { + ctx, cancel := context.WithCancel(context.Background()) + connectionObj = &ConnectionProcessor{ + UpdatedAt: time.Now(), + changes: make(map[string]*ConnectionChange), + ctx: ctx, + ctxCancel: cancel, + } + connectionProcessor[connectionID] = connectionObj + } + return connectionObj +} + +func SetError(connectionID, pathHash string, err error) { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return + } + connectionObj.lock.Lock() + connChange := connectionObj.changes[pathHash] + connChange.processError = err + connectionObj.lock.Unlock() + drainChan(connChange.ProcessChan) // drain the channel so that the no commands are blocked +} + +func GetError(connectionID, pathHash string) error { + connectionObjMutex.RLock() + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() + if connectionObj == nil { + return nil + } + connectionObj.lock.RLock() + defer connectionObj.lock.RUnlock() + connChange := connectionObj.changes[pathHash] + if connChange == nil { + return nil + } + return connChange.processError } // GetConnectionObjSize gets the connection size from the memory func GetConnectionObjSize(connectionID string) int64 { connectionObjMutex.RLock() defer connectionObjMutex.RUnlock() - connectionObjSize := connectionObjSizeMap[connectionID] - if connectionObjSize == nil { + connectionObj := connectionProcessor[connectionID] + if connectionObj == nil { return 0 } - return connectionObjSizeMap[connectionID].Size + return connectionObj.Size } // UpdateConnectionObjSize updates the connection size by addSize in memory func UpdateConnectionObjSize(connectionID string, addSize int64) { connectionObjMutex.Lock() defer connectionObjMutex.Unlock() - connectionObjSize := connectionObjSizeMap[connectionID] - if connectionObjSize == nil { - connectionObjSizeMap[connectionID] = &ConnectionObjSize{ + connectionObj := connectionProcessor[connectionID] + if connectionObj == nil { + connectionProcessor[connectionID] = &ConnectionProcessor{ Size: addSize, UpdatedAt: time.Now(), - Changes: make(map[string]*ConnectionChanges), + changes: make(map[string]*ConnectionChange), } return } - connectionObjSize.Size = connectionObjSize.Size + addSize - connectionObjSize.UpdatedAt = time.Now() + connectionObj.Size = connectionObj.Size + addSize + connectionObj.UpdatedAt = time.Now() } func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { connectionObjMutex.RLock() - defer connectionObjMutex.RUnlock() - connectionObj := connectionObjSizeMap[connectionID] + connectionObj := connectionProcessor[connectionID] + connectionObjMutex.RUnlock() if connectionObj == nil { return nil } - if connectionObj.Changes[pathHash] == nil { + connectionObj.lock.RLock() + defer connectionObj.lock.RUnlock() + if connectionObj.changes[pathHash] == nil { return nil } - return connectionObj.Changes[pathHash].Hasher + return connectionObj.changes[pathHash].hasher } func UpdateConnectionObjWithHasher(connectionID, pathHash string, hasher *filestore.CommitHasher) { connectionObjMutex.Lock() - defer connectionObjMutex.Unlock() - connectionObj := connectionObjSizeMap[connectionID] + connectionObj := connectionProcessor[connectionID] if connectionObj == nil { - connectionObjSizeMap[connectionID] = &ConnectionObjSize{ + connectionObj = &ConnectionProcessor{ UpdatedAt: time.Now(), - Changes: make(map[string]*ConnectionChanges), + changes: make(map[string]*ConnectionChange), } + connectionProcessor[connectionID] = connectionObj } - connectionObjSizeMap[connectionID].Changes[pathHash] = &ConnectionChanges{ - Hasher: hasher, + connectionObjMutex.Unlock() + connectionObj.lock.Lock() + connectionObj.changes[pathHash].hasher = hasher + connectionObj.lock.Unlock() +} + +func processCommand(ctx context.Context, processorChan chan FileCommand, allocationObj *Allocation, connectionID, clientID, pathHash string) { + + defer func() { + if r := recover(); r != nil { + logging.Logger.Error("Recovered panic", zap.String("connection_id", connectionID), zap.Any("error", r)) + SetError(connectionID, pathHash, common.NewError("panic", "Recovered panic")) + } + }() + + for { + select { + case <-ctx.Done(): + return + case cmd, ok := <-processorChan: + if cmd == nil || !ok { + return + } + res, err := cmd.ProcessContent(allocationObj) + if err != nil { + logging.Logger.Error("Error processing command", zap.String("connection_id", connectionID), zap.String("path", cmd.GetPath()), zap.Error(err)) + SetError(connectionID, pathHash, err) + return + } + err = cmd.ProcessThumbnail(allocationObj) + if err != nil && err != common.ErrNoThumbnail { + logging.Logger.Error("Error processing command", zap.String("connection_id", connectionID), zap.String("path", cmd.GetPath()), zap.Error(err)) + SetError(connectionID, pathHash, err) + return + } + if res.UpdateChange { + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + connectionObj, err := GetAllocationChanges(ctx, connectionID, allocationObj.ID, clientID) + if err != nil { + return err + } + return cmd.UpdateChange(ctx, connectionObj) + }) + if err != nil { + logging.Logger.Error("Error processing command", zap.String("connection_id", connectionID), zap.String("path", cmd.GetPath()), zap.Error(err)) + SetError(connectionID, pathHash, err) + return + } + } + if res.IsFinal { + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + connectionObj, err := GetAllocationChanges(ctx, connectionID, allocationObj.ID, clientID) + if err != nil { + return err + } + return cmd.UpdateChange(ctx, connectionObj) + }) + if err != nil { + logging.Logger.Error("Error processing command", zap.String("connection_id", connectionID), zap.String("path", cmd.GetPath()), zap.Error(err)) + SetError(connectionID, pathHash, err) + } + return + } + } + } + +} + +func drainChan(processorChan chan FileCommand) { + for { + select { + case _, ok := <-processorChan: + if !ok { + return + } + default: + return + } } } @@ -91,7 +384,11 @@ func UpdateConnectionObjWithHasher(connectionID, pathHash string, hasher *filest // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { connectionObjMutex.Lock() - delete(connectionObjSizeMap, connectionID) + connectionObj := connectionProcessor[connectionID] + if connectionObj != nil && connectionObj.ctxCancel != nil { + connectionObj.ctxCancel() + } + delete(connectionProcessor, connectionID) connectionObjMutex.Unlock() } @@ -99,10 +396,13 @@ func DeleteConnectionObjEntry(connectionID string) { // for which deadline is exceeded. func cleanConnectionObj() { connectionObjMutex.Lock() - for connectionID, connectionObjSize := range connectionObjSizeMap { - diff := time.Since(connectionObjSize.UpdatedAt) + for connectionID, connectionObj := range connectionProcessor { + diff := time.Since(connectionObj.UpdatedAt) if diff >= ConnectionObjTimeout { - delete(connectionObjSizeMap, connectionID) + if connectionObj.ctxCancel != nil { + connectionObj.ctxCancel() + } + delete(connectionProcessor, connectionID) } } connectionObjMutex.Unlock() diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 44a8455cf..8fa68ded4 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strings" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -144,7 +145,7 @@ func (rf *CopyFileChange) Unmarshal(input string) error { return err } -func (rf *CopyFileChange) CommitToFileStore(ctx context.Context) error { +func (rf *CopyFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go index 96b6da436..fcef4564a 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "sync" "testing" "time" @@ -363,7 +364,7 @@ func TestBlobberCore_CopyFile(t *testing.T) { for _, tt := range testCases { tc := tt - + mut := &sync.Mutex{} t.Run(t.Name(), func(t *testing.T) { fs := &MockFileStore{} if err := fs.Initialize(); err != nil { @@ -390,7 +391,7 @@ func TestBlobberCore_CopyFile(t *testing.T) { return err } require.Equal(t, 2, len(rootRef.Children)) - return change.CommitToFileStore(ctx) + return change.CommitToFileStore(ctx, mut) }() if tc.expectingError { diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 12babe5e6..8b0e50acd 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -58,7 +58,7 @@ func (nf *DeleteFileChange) DeleteTempFile() error { return nil } -func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { +func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { db := datastore.GetStore().GetTransaction(ctx) type Result struct { Id string @@ -69,6 +69,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { limitCh := make(chan struct{}, 10) wg := &sync.WaitGroup{} var results []Result + mut.Lock() err := db.Model(&reference.Ref{}).Unscoped(). Select("id", "validation_root", "thumbnail_hash"). Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL", @@ -115,7 +116,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error { } return nil }).Error - + mut.Unlock() wg.Wait() return err diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go index 1688e4b75..b94dc9d7b 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync" "testing" "time" @@ -242,7 +243,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) { for _, tt := range testCases { tc := tt - + mut := &sync.Mutex{} t.Run(t.Name(), func(t *testing.T) { fs := &MockFileStore{} if err := fs.Initialize(); err != nil { @@ -275,7 +276,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) { require.Equal(t, 0, len(rootRef.Children)) } - return change.CommitToFileStore(ctx) + return change.CommitToFileStore(ctx, mut) }() if tc.expectingError { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer.go b/code/go/0chain.net/blobbercore/allocation/file_changer.go index 04fe820ba..3fc8d2b7f 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" ) @@ -18,5 +19,5 @@ type FileChanger interface { // DeleteTempFile delete temp file and thumbnail from disk DeleteTempFile() error // CommitToFileStore move temp file and thumbnail from temp dir to persistent folder - CommitToFileStore(ctx context.Context) error + CommitToFileStore(ctx context.Context, mut *sync.Mutex) error } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index ca67a9f79..332d16785 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -2,8 +2,11 @@ package allocation import ( "context" + "net/http" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) @@ -56,6 +59,46 @@ type BaseFileChanger struct { ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree ChunkHash string `json:"chunk_hash,omitempty"` UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to + PathHash string `json:"-"` // hash of path +} + +// swagger:model UploadResult +type UploadResult struct { + Filename string `json:"filename"` + Size int64 `json:"size"` + Hash string `json:"hash"` + ValidationRoot string `json:"validation_root"` + FixedMerkleRoot string `json:"fixed_merkle_root"` + + // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. + UploadLength int64 `json:"upload_length"` + // Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. + UploadOffset int64 `json:"upload_offset"` + IsFinal bool `json:"-"` + UpdateChange bool `json:"-"` +} + +type FileCommand interface { + + // GetExistingFileRef get file ref if it exists + GetExistingFileRef() *reference.Ref + + GetPath() string + + // IsValidated validate request, and try build ChangeProcesser instance + IsValidated(ctx context.Context, req *http.Request, allocationObj *Allocation, clientID string) error + + // ProcessContent flush file to FileStorage + ProcessContent(allocationObj *Allocation) (UploadResult, error) + + // ProcessThumbnail flush thumbnail file to FileStorage if it has. + ProcessThumbnail(allocationObj *Allocation) error + + // UpdateChange update AllocationChangeProcessor. It will be president in db for committing transcation + UpdateChange(ctx context.Context, connectionObj *AllocationChangeCollector) error + + //NumBlocks return number of blocks uploaded by the client + GetNumBlocks() int64 } func (fc *BaseFileChanger) DeleteTempFile() error { @@ -74,7 +117,7 @@ func (fc *BaseFileChanger) DeleteTempFile() error { return err } -func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { +func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} @@ -94,6 +137,7 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { fileInputData.ValidationRoot = fc.ValidationRoot fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot fileInputData.ChunkSize = fc.ChunkSize + fileInputData.Size = fc.Size fileInputData.Hasher = GetHasher(fc.ConnectionID, encryption.Hash(fc.Path)) if fileInputData.Hasher == nil { return common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for commit.") diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 97cbdf7e0..0f501cf34 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "path/filepath" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" @@ -95,21 +96,23 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.ActualThumbnailHash = nf.ActualThumbnailHash fileRef.ActualThumbnailSize = nf.ActualThumbnailSize fileRef.EncryptedKey = nf.EncryptedKey + fileRef.EncryptedKeyPoint = nf.EncryptedKeyPoint fileRef.ChunkSize = nf.ChunkSize fileRef.IsPrecommit = true return rootRef, nil } -func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { +func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { db := datastore.GetStore().GetTransaction(ctx) for hash := range nf.deleteHash { var count int64 + mut.Lock() err := db.Table((&reference.Ref{}).TableName()). Where(&reference.Ref{ValidationRoot: hash}). Where(&reference.Ref{AllocationID: nf.AllocationID}). Count(&count).Error - + mut.Unlock() if err == nil && count == 0 { logging.Logger.Info("Deleting content file", zap.String("validation_root", hash)) if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, hash); err != nil { @@ -118,7 +121,7 @@ func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error { } } - return nf.BaseFileChanger.CommitToFileStore(ctx) + return nf.BaseFileChanger.CommitToFileStore(ctx, mut) } func (nf *UpdateFileChanger) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index dbe7bb3c3..446a51560 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -21,7 +21,7 @@ type UploadFileChanger struct { } // ApplyChange update references, and create a new FileRef -func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, +func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) @@ -101,6 +101,7 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference ActualThumbnailHash: nf.ActualThumbnailHash, ActualThumbnailSize: nf.ActualThumbnailSize, EncryptedKey: nf.EncryptedKey, + EncryptedKeyPoint: nf.EncryptedKeyPoint, ChunkSize: nf.ChunkSize, CreatedAt: ts, UpdatedAt: ts, diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go new file mode 100644 index 000000000..c9e1fa580 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go @@ -0,0 +1,28 @@ +//go:build integration_tests +// +build integration_tests + +package allocation + +import ( + "context" + "errors" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/node" +) + +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { + + state := conductrpc.Client().State() + if state.FailUploadCommit != nil { + for _, nodeId := range state.FailUploadCommit { + if nodeId == node.Self.ID { + return nil, errors.New("error directed by conductor") + } + } + } + return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +} diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go new file mode 100644 index 000000000..5a41a40aa --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -0,0 +1,15 @@ +//go:build !integration_tests +// +build !integration_tests + +package allocation + +import ( + "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { + return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +} diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index 6ee435013..a5449c2da 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -3,6 +3,7 @@ package allocation import ( "context" "path/filepath" + "sync" "testing" "time" @@ -84,7 +85,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { for _, tt := range testCases { tc := tt - + mut := &sync.Mutex{} t.Run(t.Name(), func(t *testing.T) { fs := &MockFileStore{} if err := fs.Initialize(); err != nil { @@ -101,6 +102,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { fPath := "/new" hasher := filestore.GetNewCommitHasher(2310) pathHash := encryption.Hash(fPath) + CreateConnectionChange("connection_id", pathHash, alloc) UpdateConnectionObjWithHasher("connection_id", pathHash, hasher) change := &UploadFileChanger{ BaseFileChanger: BaseFileChanger{ @@ -126,7 +128,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { return err } - return change.CommitToFileStore(ctx) + return change.CommitToFileStore(ctx, mut) }() if tc.expectingError { diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index 0b3020a16..a1dc4e46b 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strings" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -23,7 +24,7 @@ func (rf *MoveFileChange) DeleteTempFile() error { } func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { + allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { srcRef, err := rootRef.GetSrcPath(rf.SrcPath) if err != nil { @@ -33,7 +34,42 @@ func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re rootRef.UpdatedAt = ts rootRef.HashToBeComputed = true + srcParentPath, srcFileName := filepath.Split(rf.SrcPath) + srcFields, err := common.GetPathFields(srcParentPath) + if err != nil { + return nil, err + } dirRef := rootRef + for i := 0; i < len(srcFields); i++ { + found := false + for _, child := range dirRef.Children { + if child.Name == srcFields[i] { + dirRef = child + found = true + dirRef.HashToBeComputed = true + break + } + } + if !found { + return nil, common.NewError("invalid_reference_path", + fmt.Sprintf("path %s does not exist", strings.Join(srcFields[:i+1], "/"))) + } + } + + var removed bool + for i, child := range dirRef.Children { + if child.Name == srcFileName { + dirRef.RemoveChild(i) + removed = true + break + } + } + if !removed { + return nil, common.NewError("incomplete_move", + "move operation rejected as it cannot be completed") + } + + dirRef = rootRef fields, err := common.GetPathFields(rf.DestPath) if err != nil { return nil, err @@ -64,48 +100,18 @@ func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re newRef.HashToBeComputed = true newRef.CreatedAt = ts newRef.UpdatedAt = ts + fileID, ok := fileIDMeta[newRef.Path] + if !ok || fileID == "" { + return nil, common.NewError("invalid_parameter", + fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) + } + newRef.FileID = fileID dirRef.AddChild(newRef) dirRef = newRef } } - fileRefs := rf.processMoveRefs(ctx, srcRef, dirRef, allocationRoot, ts, true) - srcParentPath, srcFileName := filepath.Split(rf.SrcPath) - srcFields, err := common.GetPathFields(srcParentPath) - if err != nil { - return nil, err - } - dirRef = rootRef - for i := 0; i < len(srcFields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == srcFields[i] { - dirRef = child - found = true - dirRef.HashToBeComputed = true - break - } - } - if !found { - return nil, common.NewError("invalid_reference_path", - fmt.Sprintf("path %s does not exist", strings.Join(srcFields[:i+1], "/"))) - } - } - - var removed bool - for i, child := range dirRef.Children { - if child.Name == srcFileName { - dirRef.RemoveChild(i) - removed = true - break - } - } - if !removed { - return nil, common.NewError("incomplete_move", - "move operation rejected as it cannot be completed") - } - for _, fileRef := range fileRefs { fileRef.IsPrecommit = true } @@ -115,7 +121,6 @@ func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re func (rf *MoveFileChange) processMoveRefs( ctx context.Context, srcRef, destRef *reference.Ref, allocationRoot string, ts common.Timestamp, toAdd bool) (fileRefs []*reference.Ref) { - if srcRef.Type == reference.DIRECTORY { srcRef.Path = filepath.Join(destRef.Path, srcRef.Name) srcRef.ParentPath = destRef.Path @@ -156,7 +161,7 @@ func (rf *MoveFileChange) Unmarshal(input string) error { return err } -func (rf *MoveFileChange) CommitToFileStore(ctx context.Context) error { +func (rf *MoveFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go index ebdc70d12..b66526630 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "sync" "testing" "time" @@ -48,6 +49,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { maxDirFilesPerAlloc int expectedMessage string expectingError bool + fileIDMeta map[string]string setupDbMock func() }{ { @@ -57,6 +59,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { allocationID: alloc.ID, maxDirFilesPerAlloc: 5, expectingError: false, + fileIDMeta: map[string]string{}, setupDbMock: func() { mocket.Catcher.Reset() @@ -177,6 +180,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { allocationID: alloc.ID, maxDirFilesPerAlloc: 5, expectingError: false, + fileIDMeta: map[string]string{"/target": "file_id"}, setupDbMock: func() { mocket.Catcher.Reset() @@ -296,6 +300,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { allocationID: alloc.ID, maxDirFilesPerAlloc: 5, expectingError: false, + fileIDMeta: map[string]string{"/target": "file_id"}, setupDbMock: func() { mocket.Catcher.Reset() @@ -339,7 +344,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { "path": "/new/orig.txt", "name": "orig.txt", "allocation_id": alloc.ID, - "parent_path": "/", + "parent_path": "/new", "validation_root": "validation_root", "thumbnail_size": 00, "thumbnail_hash": "", @@ -390,7 +395,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { "path": "/new/orig.txt", "name": "orig.txt", "allocation_id": alloc.ID, - "parent_path": "/", + "parent_path": "/new", "validation_root": "validation_root", "thumbnail_size": 00, "thumbnail_hash": "", @@ -440,7 +445,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { "path": "/new/orig.txt", "name": "orig.txt", "allocation_id": alloc.ID, - "parent_path": "/", + "parent_path": "/new", "validation_root": "validation_root", "thumbnail_size": 00, "thumbnail_hash": "", @@ -456,7 +461,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { for _, tt := range testCases { tc := tt - + mut := &sync.Mutex{} t.Run(t.Name(), func(t *testing.T) { fs := &MockFileStore{} if err := fs.Initialize(); err != nil { @@ -478,12 +483,12 @@ func TestBlobberCore_MoveFile(t *testing.T) { rootRef, err := reference.GetReferencePathFromPaths(ctx, tc.allocationID, []string{change.DestPath, change.SrcPath}, []string{change.SrcPath}) require.Nil(t, err) err = func() error { - _, err := change.ApplyChange(ctx, rootRef, tc.allocChange, "/", common.Now()-1, nil) + _, err := change.ApplyChange(ctx, rootRef, tc.allocChange, "/", common.Now()-1, tc.fileIDMeta) if err != nil { return err } - return change.CommitToFileStore(ctx) + return change.CommitToFileStore(ctx, mut) }() if tc.expectingError { diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 988ee4d27..7d630b0f6 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strings" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -105,7 +106,7 @@ func (nf *NewDir) DeleteTempFile() error { return nil } -func (nfch *NewDir) CommitToFileStore(ctx context.Context) error { +func (nfch *NewDir) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index b38d0626b..716845e7f 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -9,7 +9,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" @@ -138,7 +137,36 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc logging.Logger.Info("Saving the allocation to DB") - err = Repo.Save(ctx, a) + if !isExist { + err = Repo.Save(ctx, a) + } else { + updateMap := map[string]interface{}{ + "tx": a.Tx, + "expiration_date": a.Expiration, + "owner_id": a.OwnerID, + "owner_public_key": a.OwnerPublicKey, + "repairer_id": a.RepairerID, + "size": a.TotalSize, + "finalized": a.Finalized, + "time_unit": a.TimeUnit, + "file_options": a.FileOptions, + "start_time": a.StartTime, + } + + updateOption := func(alloc *Allocation) { + alloc.Tx = a.Tx + alloc.Expiration = a.Expiration + alloc.OwnerID = a.OwnerID + alloc.OwnerPublicKey = a.OwnerPublicKey + alloc.RepairerID = a.RepairerID + alloc.TotalSize = a.TotalSize + alloc.Finalized = a.Finalized + alloc.TimeUnit = a.TimeUnit + alloc.FileOptions = a.FileOptions + alloc.StartTime = a.StartTime + } + err = Repo.UpdateAllocation(ctx, a, updateMap, updateOption) + } if err != nil { return nil, err @@ -161,7 +189,7 @@ func RequestReadPoolStat(clientID string) (*ReadPool, error) { params := map[string]string{ "client_id": clientID, } - resp, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/getReadPoolStat", params, chain.GetServerChain()) + resp, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/getReadPoolStat", params) if err != nil { return nil, fmt.Errorf("requesting read pools stat: %v", err) } @@ -188,7 +216,7 @@ func RequestWritePool(allocationID string) (wps *WritePool, err error) { params := map[string]string{ "allocation": allocationID, } - resp, err = transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/allocation", params, chain.GetServerChain()) + resp, err = transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/allocation", params) if err != nil { return nil, fmt.Errorf("requesting write pools stat: %v", err) } diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index 75145e6e3..fbcee3759 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "path/filepath" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -18,13 +19,14 @@ type RenameFileChange struct { Path string `json:"path"` NewName string `json:"new_name"` Name string `json:"name"` + Type string `json:"type"` } func (rf *RenameFileChange) DeleteTempFile() error { return nil } -func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, +func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { if rf.Path == "/" { @@ -126,11 +128,13 @@ func (rf *RenameFileChange) Unmarshal(input string) error { return err } -func (rf *RenameFileChange) CommitToFileStore(ctx context.Context) error { +func (rf *RenameFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { return nil } func (rf *RenameFileChange) GetPath() []string { - + if rf.Type == reference.DIRECTORY { + return []string{rf.Path, rf.Path} + } return []string{rf.Path} } diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_integration.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_integration.go new file mode 100644 index 000000000..1e23b568a --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_integration.go @@ -0,0 +1,28 @@ +//go:build integration_tests +// +build integration_tests + +package allocation + +import ( + "context" + "errors" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/node" +) + +func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { + + state := conductrpc.Client().State() + if state.FailRenameCommit != nil { + for _, nodeId := range state.FailRenameCommit { + if nodeId == node.Self.ID { + return nil, errors.New("error directed by conductor") + } + } + } + return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) +} diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go new file mode 100644 index 000000000..9c773cab7 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go @@ -0,0 +1,17 @@ +//go:build !integration_tests +// +build !integration_tests + +package allocation + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, + allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { + + return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) +} diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index 2be71a223..c895c7003 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -108,7 +108,7 @@ func setupMockForFileManagerInit(mock sqlmock.Sqlmock) { WillReturnRows( sqlmock.NewRows([]string{"file_size"}).AddRow(6553600), ) - + mock.ExpectCommit() } func init() { diff --git a/code/go/0chain.net/blobbercore/allocation/repository.go b/code/go/0chain.net/blobbercore/allocation/repository.go index e6dddda9f..52d2f1253 100644 --- a/code/go/0chain.net/blobbercore/allocation/repository.go +++ b/code/go/0chain.net/blobbercore/allocation/repository.go @@ -3,9 +3,11 @@ package allocation import ( "context" "fmt" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/logging" + lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -14,17 +16,32 @@ import ( const ( SQLWhereGetById = "allocations.id = ?" SQLWhereGetByTx = "allocations.tx = ?" + lruSize = 100 ) var ( - Repo *Repository + Repo *Repository + mapLock sync.Mutex ) +type AllocationUpdate func(a *Allocation) + func init() { - Repo = &Repository{} + allocCache, _ := lru.New[string, Allocation](lruSize) + Repo = &Repository{ + allocCache: allocCache, + allocLock: make(map[string]*sync.Mutex), + } } type Repository struct { + allocCache *lru.Cache[string, Allocation] + allocLock map[string]*sync.Mutex +} + +type AllocationCache struct { + Allocation *Allocation + AllocationUpdates []AllocationUpdate } type Res struct { @@ -43,6 +60,14 @@ func (r *Repository) GetById(ctx context.Context, id string) (*Allocation, error } if a, ok := cache[id]; ok { + return a.Allocation, nil + } + + a := r.getAllocFromGlobalCache(id) + if a != nil { + cache[id] = AllocationCache{ + Allocation: a, + } return a, nil } @@ -52,8 +77,10 @@ func (r *Repository) GetById(ctx context.Context, id string) (*Allocation, error return alloc, err } - cache[id] = alloc - + cache[id] = AllocationCache{ + Allocation: alloc, + } + r.setAllocToGlobalCache(alloc) return alloc, nil } @@ -69,7 +96,6 @@ func (r *Repository) GetByIdAndLock(ctx context.Context, id string) (*Allocation } alloc := &Allocation{} - err = tx.Model(&Allocation{}). Clauses(clause.Locking{Strength: "NO KEY UPDATE"}). Where("id=?", id). @@ -77,8 +103,9 @@ func (r *Repository) GetByIdAndLock(ctx context.Context, id string) (*Allocation if err != nil { return alloc, err } - cache[id] = alloc - + cache[id] = AllocationCache{ + Allocation: alloc, + } return alloc, err } @@ -93,18 +120,28 @@ func (r *Repository) GetByTx(ctx context.Context, allocationID, txHash string) ( return nil, err } if a, ok := cache[allocationID]; ok { - if a.Tx == txHash { - return a, nil + if a.Allocation.Tx == txHash { + return a.Allocation, nil } } + a := r.getAllocFromGlobalCache(allocationID) + if a != nil && a.Tx == txHash { + cache[allocationID] = AllocationCache{ + Allocation: a, + } + return a, nil + } + alloc := &Allocation{} err = tx.Table(TableNameAllocation).Where(SQLWhereGetByTx, txHash).Take(alloc).Error if err != nil { return alloc, err } - cache[allocationID] = alloc - + cache[allocationID] = AllocationCache{ + Allocation: alloc, + } + r.setAllocToGlobalCache(alloc) return alloc, err } @@ -112,13 +149,28 @@ func (r *Repository) GetAllocations(ctx context.Context, offset int64) ([]*Alloc var tx = datastore.GetStore().GetTransaction(ctx) const query = `finalized = false AND cleaned_up = false` - allocs := make([]*Allocation, 0) - return allocs, tx.Model(&Allocation{}). + allocs := make([]*Allocation, 0, 10) + err := tx.Model(&Allocation{}). Where(query). Limit(UPDATE_LIMIT). Offset(int(offset)). Order("id ASC"). Find(&allocs).Error + if err != nil { + return allocs, err + } + return allocs, nil +} + +func (r *Repository) GetAllocationFromDB(ctx context.Context, allocationID string) (*Allocation, error) { + var tx = datastore.GetStore().GetTransaction(ctx) + + alloc := &Allocation{} + err := tx.Model(&Allocation{}).Where("id = ?", allocationID).Take(alloc).Error + if err != nil { + return nil, err + } + return alloc, nil } func (r *Repository) GetAllocationIds(ctx context.Context) []Res { @@ -149,16 +201,78 @@ func (r *Repository) UpdateAllocationRedeem(ctx context.Context, allocationID, A if err != nil { return err } - delete(cache, allocationID) allocationUpdates := make(map[string]interface{}) allocationUpdates["latest_redeemed_write_marker"] = AllocationRoot allocationUpdates["is_redeem_required"] = false err = tx.Model(allocationObj).Updates(allocationUpdates).Error - return err + if err != nil { + return err + } + allocationObj.LatestRedeemedWM = AllocationRoot + allocationObj.IsRedeemRequired = false + txnCache := cache[allocationID] + txnCache.Allocation = allocationObj + updateAlloc := func(a *Allocation) { + a.LatestRedeemedWM = AllocationRoot + a.IsRedeemRequired = false + } + txnCache.AllocationUpdates = append(txnCache.AllocationUpdates, updateAlloc) + cache[allocationID] = txnCache + return nil +} + +func (r *Repository) UpdateAllocation(ctx context.Context, allocationObj *Allocation, updateMap map[string]interface{}, updateOption AllocationUpdate) error { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + cache, err := getCache(tx) + if err != nil { + return err + } + err = tx.Model(allocationObj).Updates(updateMap).Error + if err != nil { + return err + } + txnCache := cache[allocationObj.ID] + txnCache.Allocation = allocationObj + txnCache.AllocationUpdates = append(txnCache.AllocationUpdates, updateOption) + cache[allocationObj.ID] = txnCache + return nil } -func (r *Repository) Save(ctx context.Context, a *Allocation) error { +func (r *Repository) Commit(tx *datastore.EnhancedDB) { + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + cache, _ := getCache(tx) + if cache == nil { + return + } + for _, txnCache := range cache { + alloc := r.getAllocFromGlobalCache(txnCache.Allocation.ID) + mapLock.Lock() + mut, ok := r.allocLock[txnCache.Allocation.ID] + if !ok { + mut = &sync.Mutex{} + r.allocLock[txnCache.Allocation.ID] = mut + } + mapLock.Unlock() + mut.Lock() + if alloc != nil { + for _, update := range txnCache.AllocationUpdates { + update(alloc) + } + if len(txnCache.AllocationUpdates) > 0 { + r.setAllocToGlobalCache(alloc) + } + } + mut.Unlock() + } +} + +func (r *Repository) Save(ctx context.Context, alloc *Allocation) error { var tx = datastore.GetStore().GetTransaction(ctx) if tx == nil { logging.Logger.Panic("no transaction in the context") @@ -169,20 +283,73 @@ func (r *Repository) Save(ctx context.Context, a *Allocation) error { return err } - cache[a.ID] = a - return tx.Save(a).Error + txnCache := cache[alloc.ID] + txnCache.Allocation = alloc + err = tx.Save(alloc).Error + if err != nil { + return err + } + updateAlloc := func(a *Allocation) { + *a = *alloc + } + txnCache.AllocationUpdates = append(txnCache.AllocationUpdates, updateAlloc) + cache[alloc.ID] = txnCache + return nil +} + +func (r *Repository) Create(ctx context.Context, alloc *Allocation) error { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + cache, err := getCache(tx) + if err != nil { + return err + } + + txnCache := cache[alloc.ID] + txnCache.Allocation = alloc + err = tx.Create(alloc).Error + if err != nil { + return err + } + cache[alloc.ID] = txnCache + return nil } -func getCache(tx *datastore.EnhancedDB) (map[string]*Allocation, error) { +func getCache(tx *datastore.EnhancedDB) (map[string]AllocationCache, error) { c, ok := tx.SessionCache[TableNameAllocation] if ok { - cache, ok := c.(map[string]*Allocation) + cache, ok := c.(map[string]AllocationCache) if !ok { return nil, fmt.Errorf("type assertion failed") } return cache, nil } - cache := make(map[string]*Allocation) + cache := make(map[string]AllocationCache) tx.SessionCache[TableNameAllocation] = cache + if tx.CommitAllocCache == nil { + tx.CommitAllocCache = func(tx *datastore.EnhancedDB) { + Repo.Commit(tx) + } + } return cache, nil } + +func (r *Repository) getAllocFromGlobalCache(id string) *Allocation { + a, ok := r.allocCache.Get(id) + if !ok { + return nil + } + return &a +} + +func (r *Repository) setAllocToGlobalCache(a *Allocation) { + if a != nil { + r.allocCache.Add(a.ID, *a) + } +} + +func (r *Repository) DeleteAllocation(allocationID string) { + r.allocCache.Remove(allocationID) +} diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index 4452299ef..02f2113d8 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "sync" "testing" "time" @@ -340,6 +341,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { ctx := datastore.GetStore().CreateTransaction(context.TODO()) hasher := filestore.GetNewCommitHasher(2310) pathHash := encryption.Hash(tc.path) + CreateConnectionChange("connection_id", pathHash, alloc) UpdateConnectionObjWithHasher("connection_id", pathHash, hasher) change := &UpdateFileChanger{ @@ -359,7 +361,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { ConnectionID: "connection_id", }, } - + mut := &sync.Mutex{} t.Run(tc.name, func(t *testing.T) { rootRef, _ := reference.GetReferencePathFromPaths(ctx, tc.allocationID, []string{change.Path}, []string{}) _, err := func() (*reference.Ref, error) { @@ -368,7 +370,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { return nil, err } - err = change.CommitToFileStore(ctx) + err = change.CommitToFileStore(ctx, mut) return resp, err }() diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index 44306b911..ff19d4130 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -3,12 +3,14 @@ package allocation import ( "context" "encoding/json" + "math" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" @@ -27,6 +29,10 @@ func StartUpdateWorker(ctx context.Context, interval time.Duration) { go UpdateWorker(ctx, interval) } +func StartFinalizeWorker(ctx context.Context, interval time.Duration) { + go FinalizeAllocationsWorker(ctx, interval) +} + // UpdateWorker updates all not finalized and not cleaned allocations // requesting SC through REST API. The worker required to fetch allocations // updates in DB. @@ -44,10 +50,36 @@ func UpdateWorker(ctx context.Context, interval time.Duration) { for { select { case <-tick: - _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + updateCtx := datastore.GetStore().CreateTransaction(context.TODO()) + _ = datastore.GetStore().WithTransaction(updateCtx, func(ctx context.Context) error { updateWork(ctx) return nil }) + updateCtx.Done() + case <-quit: + return + } + } +} + +func FinalizeAllocationsWorker(ctx context.Context, interval time.Duration) { + logging.Logger.Info("start finalize allocations worker") + + var tk = time.NewTicker(interval) + defer tk.Stop() + + var ( + tick = tk.C + quit = ctx.Done() + ) + + for { + select { + case <-tick: + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + finalizeExpiredAllocations(ctx) + return nil + }) case <-quit: return } @@ -101,7 +133,7 @@ func updateWork(ctx context.Context) { offset += int64(len(allocs)) for _, a := range allocs { - updateAllocation(ctx, a) + updateAllocation(ctx, a, node.Self.ID) if waitOrQuit(ctx, REQUEST_TIMEOUT) { return } @@ -117,10 +149,10 @@ func findAllocations(ctx context.Context, offset int64) (allocs []*Allocation, c func shouldFinalize(sa *transaction.StorageAllocation) bool { var now = common.Now() - return sa.Until() < now && !sa.Finalized + return sa.Expiration < now && !sa.Finalized } -func updateAllocation(ctx context.Context, a *Allocation) { +func updateAllocation(ctx context.Context, a *Allocation, selfBlobberID string) { if a.Finalized { cleanupAllocation(ctx, a) return @@ -132,6 +164,19 @@ func updateAllocation(ctx context.Context, a *Allocation) { return } + removedBlobber := true + for _, d := range sa.BlobberDetails { + if d.BlobberID == selfBlobberID { + removedBlobber = false + break + } + } + + if removedBlobber { + logging.Logger.Info("blobber removed from allocation", zap.String("blobber", selfBlobberID), zap.String("allocation", a.ID)) + cleanupAllocation(ctx, a) + } + // if new Tx, then we have to update the allocation if sa.Tx != a.Tx || sa.OwnerID != a.OwnerID || sa.Finalized != a.Finalized { if a, err = updateAllocationInDB(ctx, a, sa); err != nil { @@ -142,7 +187,8 @@ func updateAllocation(ctx context.Context, a *Allocation) { // send finalize allocation transaction if shouldFinalize(sa) { - sendFinalizeAllocation(a) + sendFinalizeAllocation(a.ID) + cleanupAllocation(ctx, a) return } @@ -152,13 +198,24 @@ func updateAllocation(ctx context.Context, a *Allocation) { } } +func finalizeExpiredAllocations(ctx context.Context) { + var allocs, err = requestExpiredAllocations() + if err != nil { + logging.Logger.Error("requesting expired allocations from SC", zap.Error(err)) + return + } + + for _, allocID := range allocs { + sendFinalizeAllocation(allocID) + } +} + func requestAllocation(allocID string) (sa *transaction.StorageAllocation, err error) { var b []byte b, err = transaction.MakeSCRestAPICall( transaction.STORAGE_CONTRACT_ADDRESS, "/allocation", - map[string]string{"allocation": allocID}, - chain.GetServerChain()) + map[string]string{"allocation": allocID}) if err != nil { return } @@ -167,20 +224,58 @@ func requestAllocation(allocID string) (sa *transaction.StorageAllocation, err e return } +func requestExpiredAllocations() (allocs []string, err error) { + var b []byte + b, err = transaction.MakeSCRestAPICall( + transaction.STORAGE_CONTRACT_ADDRESS, + "/expired-allocations", + map[string]string{"blobber_id": node.Self.ID}) + if err != nil { + return + } + err = json.Unmarshal(b, &allocs) + return +} + func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.StorageAllocation) (ua *Allocation, err error) { var tx = datastore.GetStore().GetTransaction(ctx) var changed bool = a.Tx != sa.Tx + if !changed { + return a, nil + } // transaction a.Tx = sa.Tx a.OwnerID = sa.OwnerID a.OwnerPublicKey = sa.OwnerPublicKey - // update fields + // // update fields a.Expiration = sa.Expiration a.TotalSize = sa.Size a.Finalized = sa.Finalized a.FileOptions = sa.FileOptions + a.BlobberSize = int64(math.Ceil(float64(sa.Size) / float64(sa.DataShards))) + + updateMap := make(map[string]interface{}) + updateMap["tx"] = a.Tx + updateMap["owner_id"] = a.OwnerID + updateMap["owner_public_key"] = a.OwnerPublicKey + updateMap["expiration_date"] = a.Expiration + updateMap["size"] = a.TotalSize + updateMap["finalized"] = a.Finalized + updateMap["file_options"] = a.FileOptions + updateMap["blobber_size"] = a.BlobberSize + + updateOption := func(alloc *Allocation) { + alloc.Tx = a.Tx + alloc.OwnerID = a.OwnerID + alloc.OwnerPublicKey = a.OwnerPublicKey + alloc.Expiration = a.Expiration + alloc.TotalSize = a.TotalSize + alloc.Finalized = a.Finalized + alloc.FileOptions = a.FileOptions + alloc.BlobberSize = a.BlobberSize + } // update terms a.Terms = make([]*Terms, 0, len(sa.BlobberDetails)) @@ -194,14 +289,10 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St } // save allocations - if err := Repo.Save(ctx, a); err != nil { + if err := Repo.UpdateAllocation(ctx, a, updateMap, updateOption); err != nil { return nil, err } - if !changed { - return a, nil - } - // save allocation terms for _, t := range a.Terms { if err := tx.Save(t).Error; err != nil { @@ -217,7 +308,7 @@ type finalizeRequest struct { AllocationID string `json:"allocation_id"` } -func sendFinalizeAllocation(a *Allocation) { +func sendFinalizeAllocation(allocationID string) { var tx, err = transaction.NewTransactionEntity() if err != nil { logging.Logger.Error("creating new transaction entity", zap.Error(err)) @@ -225,9 +316,8 @@ func sendFinalizeAllocation(a *Allocation) { } var request finalizeRequest - request.AllocationID = a.ID + request.AllocationID = allocationID - // TODO should this be verified? err = tx.ExecuteSmartContract( transaction.STORAGE_CONTRACT_ADDRESS, transaction.FINALIZE_ALLOCATION, diff --git a/code/go/0chain.net/blobbercore/allocation/zcn.go b/code/go/0chain.net/blobbercore/allocation/zcn.go index bc7b0c463..c5d419ecf 100644 --- a/code/go/0chain.net/blobbercore/allocation/zcn.go +++ b/code/go/0chain.net/blobbercore/allocation/zcn.go @@ -66,7 +66,7 @@ func SyncAllocation(allocationId string) (*Allocation, error) { err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { var e error - if e := Repo.Save(ctx, alloc); e != nil { + if e := Repo.Create(ctx, alloc); e != nil { return e } tx := datastore.GetStore().GetTransaction(ctx) diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index dc0ecd069..b48fcdbd6 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -1,27 +1,12 @@ package blobberhttp import ( - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -// swagger:model UploadResult -type UploadResult struct { - Filename string `json:"filename"` - Size int64 `json:"size"` - Hash string `json:"hash"` - ValidationRoot string `json:"validation_root"` - FixedMerkleRoot string `json:"fixed_merkle_root"` - - // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. - UploadLength int64 `json:"upload_length"` - // Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. - UploadOffset int64 `json:"upload_offset"` -} - // swagger:model ConnectionResult type ConnectionResult struct { AllocationRoot string `json:"allocation_root"` @@ -30,11 +15,10 @@ type ConnectionResult struct { // swagger:model CommitResult type CommitResult struct { - AllocationRoot string `json:"allocation_root"` - WriteMarker *writemarker.WriteMarker `json:"write_marker"` - Success bool `json:"success"` - ErrorMessage string `json:"error_msg,omitempty"` - Changes []*allocation.AllocationChange `json:"-"` + AllocationRoot string `json:"allocation_root"` + WriteMarker *writemarker.WriteMarker `json:"write_marker"` + Success bool `json:"success"` + ErrorMessage string `json:"error_msg,omitempty"` //Result []*UploadResult `json:"result"` } diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 04aea1eea..afa9fa4bc 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -8,10 +8,10 @@ import ( "strconv" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "go.uber.org/zap" @@ -32,14 +32,6 @@ func syncOpenChallenges(ctx context.Context) { } }() - params := make(map[string]string) - params["blobber"] = node.Self.ID - - params["limit"] = "50" - if lastChallengeRound > 0 { - params["from"] = strconv.FormatInt(lastChallengeRound, 10) - } - start := time.Now() var downloadElapsed, jsonElapsed time.Duration @@ -52,13 +44,19 @@ func syncOpenChallenges(ctx context.Context) { default: } + params := make(map[string]string) + params["blobber"] = node.Self.ID + + params["limit"] = "20" + params["from"] = strconv.FormatInt(lastChallengeRound, 10) + logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) var challenges BCChallengeResponse var challengeIDs []string challenges.Challenges = make([]*ChallengeEntity, 0) apiStart := time.Now() - retBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/openchallenges", params, chain.GetServerChain()) + retBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/openchallenges", params) if err != nil { logging.Logger.Error("[challenge]open: ", zap.Error(err)) break @@ -145,7 +143,6 @@ func validateOnValidators(ctx context.Context, c *ChallengeEntity) error { zap.Any("challenge_id", c.ChallengeID), zap.Time("created", createdTime), zap.Error(err)) - //TODO: Should we delete the challenge from map or send it back to the todo channel? deleteChallenge(c.RoundCreatedAt) return err } @@ -167,12 +164,24 @@ func validateOnValidators(ctx context.Context, c *ChallengeEntity) error { func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transaction.Transaction, error) { createdTime := common.ToTime(c.CreatedAt) - logging.Logger.Info("[challenge]commit", + + logging.Logger.Info("[challenge]verify: ", zap.Any("challenge_id", c.ChallengeID), - zap.Time("created", createdTime), - zap.Any("openchallenge", c)) + zap.Time("created", createdTime)) + + currentRound := roundInfo.CurrentRound + int64(float64(roundInfo.LastRoundDiff)*(float64(time.Since(roundInfo.CurrentRoundCaptureTime).Milliseconds())/float64(GetRoundInterval.Milliseconds()))) + logging.Logger.Info("[challenge]commit", + zap.Any("ChallengeID", c.ChallengeID), + zap.Any("RoundCreatedAt", c.RoundCreatedAt), + zap.Any("ChallengeCompletionTime", config.StorageSCConfig.ChallengeCompletionTime), + zap.Any("currentRound", currentRound), + zap.Any("roundInfo.LastRoundDiff", roundInfo.LastRoundDiff), + zap.Any("roundInfo.CurrentRound", roundInfo.CurrentRound), + zap.Any("roundInfo.CurrentRoundCaptureTime", roundInfo.CurrentRoundCaptureTime), + zap.Any("time.Since(roundInfo.CurrentRoundCaptureTime).Milliseconds()", time.Since(roundInfo.CurrentRoundCaptureTime).Milliseconds()), + ) - if time.Since(common.ToTime(c.CreatedAt)) > config.StorageSCConfig.ChallengeCompletionTime { + if currentRound-c.RoundCreatedAt > config.StorageSCConfig.ChallengeCompletionTime { c.CancelChallenge(ctx, ErrExpiredCCT) return nil, nil } diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index 6218673fa..ab95dec3e 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" + "sync" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -48,6 +50,11 @@ const ( ChallengeFailure ) +const ( + cleanupInterval = 30 * time.Minute + cleanupGap = 1 * time.Hour +) + type ValidationTicket struct { ChallengeID string `json:"challenge_id"` BlobberID string `json:"blobber_id"` @@ -100,6 +107,7 @@ type ChallengeEntity struct { RoundCreatedAt int64 `gorm:"round_created_at" json:"round_created_at"` CreatedAt common.Timestamp `gorm:"created_at" json:"created"` UpdatedAt time.Time `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"` + statusMutex *sync.Mutex `gorm:"-" json:"-"` } func (ChallengeEntity) TableName() string { @@ -206,3 +214,24 @@ func GetChallengeEntity(ctx context.Context, challengeID string) (*ChallengeEnti } return cr, nil } + +func SetupChallengeCleanUpWorker(ctx context.Context) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(cleanupInterval): + cleanUpWorker() + } + } + }() +} + +func cleanUpWorker() { + currentRound := roundInfo.CurrentRound + int64(float64(roundInfo.LastRoundDiff)*(float64(time.Since(roundInfo.CurrentRoundCaptureTime).Milliseconds())/float64(GetRoundInterval.Milliseconds()))) + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + return db.Model(&ChallengeEntity{}).Unscoped().Delete(&ChallengeEntity{}, "status <> ? AND round_created_at < ?", Cancelled, currentRound-config.Configuration.ChallengeCleanupGap).Error + }) +} diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 017e737d4..fbb0610df 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -49,7 +49,9 @@ func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) cancellation := time.Now() db := datastore.GetStore().GetTransaction(ctx) deleteChallenge(cr.RoundCreatedAt) + cr.statusMutex.Lock() cr.Status = Cancelled + cr.statusMutex.Unlock() cr.StatusMessage = errReason.Error() cr.UpdatedAt = cancellation.UTC() if err := db.Save(cr).Error; err != nil { @@ -62,7 +64,7 @@ func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) zap.Time("cancellation", cancellation), zap.Error(err)) } - logging.Logger.Error("[challenge]canceled", zap.String("challenge_id", cr.ChallengeID), zap.Error(errReason)) + logging.Logger.Error("[challenge]canceled", zap.String("challenge_id", cr.ChallengeID), zap.Any("round_created_at", cr.RoundCreatedAt), zap.Error(errReason)) } // LoadValidationTickets load validation tickets @@ -79,7 +81,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { allocMu := lock.GetMutex(allocation.Allocation{}.TableName(), cr.AllocationID) allocMu.RLock() - allocationObj, err := allocation.GetAllocationByID(ctx, cr.AllocationID) + allocationObj, err := allocation.Repo.GetAllocationFromDB(ctx, cr.AllocationID) if err != nil { allocMu.RUnlock() cr.CancelChallenge(ctx, err) @@ -106,6 +108,9 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { blockNum := int64(0) var objectPath *reference.ObjectPath if rootRef != nil { + if rootRef.Hash != allocationObj.AllocationRoot { + logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("latest_write_marker", wms[len(wms)-1].WM.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash)) + } if rootRef.NumBlocks > 0 { r := rand.New(rand.NewSource(cr.RandomNumber)) blockNum = r.Int63n(rootRef.NumBlocks) @@ -120,9 +125,10 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { cr.CancelChallenge(ctx, err) return err } - - cr.RefID = objectPath.RefID - cr.ObjectPath = objectPath + if objectPath != nil { + cr.RefID = objectPath.RefID + cr.ObjectPath = objectPath + } } cr.RespondedAllocationRoot = allocationObj.AllocationRoot @@ -225,7 +231,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { cr.ValidationTickets = make([]*ValidationTicket, len(cr.Validators)) } - accessMu := sync.Mutex{} + accessMu := sync.RWMutex{} updateMapAndSlice := func(validatorID string, i int, vt *ValidationTicket) { accessMu.Lock() cr.ValidationTickets[i] = vt @@ -237,6 +243,9 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { accessMu.Unlock() } + numSuccess := 0 + numFailed := 0 + swg := sizedwaitgroup.New(10) for i, validator := range cr.Validators { if cr.ValidationTickets[i] != nil { @@ -254,7 +263,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { resp, err := util.SendPostRequest(url, postDataBytes, nil) if err != nil { - //network issue, don't cancel it, and try it again + numFailed++ logging.Logger.Error("[challenge]post: ", zap.Any("error", err.Error())) updateMapAndSlice(validatorID, i, nil) return @@ -262,6 +271,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { var validationTicket ValidationTicket err = json.Unmarshal(resp, &validationTicket) if err != nil { + numFailed++ logging.Logger.Error( "[challenge]resp: ", zap.String("validator", @@ -272,12 +282,17 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { updateMapAndSlice(validatorID, i, nil) return } + logging.Logger.Info( "[challenge]resp: Got response from the validator.", zap.Any("validator_response", validationTicket), + zap.Any("numSuccess", numSuccess), + zap.Any("numFailed", numFailed), ) + verified, err := validationTicket.VerifySign() if err != nil || !verified { + numFailed++ logging.Logger.Error( "[challenge]ticket: Validation ticket from validator could not be verified.", zap.String("validator", validatorID), @@ -286,38 +301,25 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { return } updateMapAndSlice(validatorID, i, &validationTicket) + + numSuccess++ }(url, validator.ID, i) } - swg.Wait() - - numSuccess := 0 - numFailure := 0 - - numValidatorsResponded := 0 - for _, vt := range cr.ValidationTickets { - if vt != nil { - if vt.Result { - numSuccess++ - } else { - logging.Logger.Error("[challenge]ticket: "+vt.Message, zap.String("validator", vt.ValidatorID)) - numFailure++ - } - numValidatorsResponded++ + for { + if numSuccess > (len(cr.Validators)/2) || numSuccess+numFailed == len(cr.Validators) { + break } } - logging.Logger.Info("[challenge]validator response stats", zap.Any("challenge_id", cr.ChallengeID), zap.Any("validator_responses", responses)) - if numSuccess > (len(cr.Validators)/2) || numFailure > (len(cr.Validators)/2) || numValidatorsResponded == len(cr.Validators) { - if numSuccess > (len(cr.Validators) / 2) { - cr.Result = ChallengeSuccess - } else { - cr.Result = ChallengeFailure - - logging.Logger.Error("[challenge]validate: ", zap.String("challenge_id", cr.ChallengeID), zap.Any("block_num", cr.BlockNum), zap.Any("object_path", objectPath)) - } - + accessMu.RLock() + logging.Logger.Info("[challenge]validator response stats", zap.Any("challenge_id", cr.ChallengeID), zap.Any("validator_responses", responses), zap.Any("num_success", numSuccess), zap.Any("num_failed", numFailed)) + accessMu.RUnlock() + if numSuccess > (len(cr.Validators) / 2) { + cr.Result = ChallengeSuccess + cr.statusMutex.Lock() cr.Status = Processed + cr.statusMutex.Unlock() cr.UpdatedAt = time.Now().UTC() } else { cr.CancelChallenge(ctx, ErrNoConsensusChallenge) @@ -384,7 +386,9 @@ func IsEntityNotFoundError(err error) bool { } func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *transaction.Transaction, toAdd bool) { + cr.statusMutex.Lock() cr.Status = Committed + cr.statusMutex.Unlock() cr.StatusMessage = t.TransactionOutput cr.CommitTxnID = t.Hash cr.UpdatedAt = time.Now().UTC() diff --git a/code/go/0chain.net/blobbercore/challenge/timing.go b/code/go/0chain.net/blobbercore/challenge/timing.go index ca7a78c20..1f0b800f4 100644 --- a/code/go/0chain.net/blobbercore/challenge/timing.go +++ b/code/go/0chain.net/blobbercore/challenge/timing.go @@ -3,6 +3,7 @@ package challenge import ( "context" "fmt" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -186,3 +187,24 @@ func GetChallengeTiming(challengeID string) (*ChallengeTiming, error) { }) return ch, err } + +func SetupChallengeTimingsCleanupWorker(ctx context.Context) { + + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(cleanupInterval): + cleanUpTimingWorker() + } + } + }() +} + +func cleanUpTimingWorker() { + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + return db.Model(&ChallengeTiming{}).Unscoped().Delete(&ChallengeTiming{}, "created_at_blobber < ?", time.Now().Add(-cleanupGap).Unix()).Error + }) +} diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index 3c7adbbbb..b721f2030 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/0chain/gosdk/zcncore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -14,12 +16,20 @@ import ( "golang.org/x/sync/semaphore" ) +const GetRoundInterval = 3 * time.Minute + type TodoChallenge struct { Id string CreatedAt time.Time Status ChallengeStatus } +type RoundInfo struct { + CurrentRound int64 + CurrentRoundCaptureTime time.Time + LastRoundDiff int64 +} + func Int64Comparator(a, b interface{}) int { aAsserted := a.(int64) bAsserted := b.(int64) @@ -37,6 +47,7 @@ var ( toProcessChallenge = make(chan *ChallengeEntity, 100) challengeMap = treemap.NewWith(Int64Comparator) challengeMapLock = sync.RWMutex{} + roundInfo = RoundInfo{} ) const batchSize = 5 @@ -60,12 +71,35 @@ func startPullWorker(ctx context.Context) { } func startWorkers(ctx context.Context) { + go getRoundWorker(ctx) + // start challenge listeners go challengeProcessor(ctx) go commitOnChainWorker(ctx) } +func getRoundWorker(ctx context.Context) { + ticker := time.NewTicker(GetRoundInterval) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + currentRound, _ := zcncore.GetRoundFromSharders() + + if roundInfo.LastRoundDiff == 0 { + roundInfo.LastRoundDiff = 1000 + } else { + roundInfo.LastRoundDiff = currentRound - roundInfo.CurrentRound + } + roundInfo.CurrentRound = currentRound + roundInfo.CurrentRoundCaptureTime = time.Now() + } + } +} + func challengeProcessor(ctx context.Context) { defer func() { if r := recover(); r != nil { @@ -120,7 +154,7 @@ func processChallenge(ctx context.Context, it *ChallengeEntity) { func commitOnChainWorker(ctx context.Context) { defer func() { if r := recover(); r != nil { - logging.Logger.Error("[commitWorker]challenge", zap.Any("err", r)) + logging.Logger.Error("[commitWorker]challenge recovery", zap.Any("err", r)) } }() wg := sync.WaitGroup{} @@ -163,7 +197,7 @@ func commitOnChainWorker(ctx context.Context) { _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { err := challenge.VerifyChallengeTransaction(ctx, txn) if err == nil || err != ErrEntityNotFound { - deleteChallenge(int64(challenge.RoundCreatedAt)) + deleteChallenge(challenge.RoundCreatedAt) } return nil }) @@ -176,23 +210,37 @@ func commitOnChainWorker(ctx context.Context) { func getBatch(batchSize int) (chall []ChallengeEntity) { challengeMapLock.RLock() - defer challengeMapLock.RUnlock() if challengeMap.Size() == 0 { + challengeMapLock.RUnlock() return } + var toClean []int64 it := challengeMap.Iterator() for it.Next() { if len(chall) >= batchSize { break } ticket := it.Value().(*ChallengeEntity) - if ticket.Status != Processed { - break + ticket.statusMutex.Lock() + switch ticket.Status { + case Accepted: + ticket.statusMutex.Unlock() + goto brekLoop + case Processed: + chall = append(chall, *ticket) + default: + toClean = append(toClean, ticket.RoundCreatedAt) } - chall = append(chall, *ticket) + ticket.statusMutex.Unlock() } +brekLoop: + challengeMapLock.RUnlock() + for _, r := range toClean { + deleteChallenge(r) + } + return } @@ -213,6 +261,8 @@ func (it *ChallengeEntity) createChallenge(ctx context.Context) bool { logging.Logger.Info("createChallenge", zap.String("challenge_id", it.ChallengeID), zap.String("status", "already exists")) return false } + it.statusMutex = &sync.Mutex{} + it.Status = Accepted challengeMap.Put(it.RoundCreatedAt, it) return true } diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index f1ac23ecc..75f6835b2 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -2,10 +2,12 @@ package config import ( "fmt" + "log" "strings" "time" "github.com/0chain/blobber/code/go/0chain.net/core/config" + "github.com/fsnotify/fsnotify" "github.com/spf13/viper" ) @@ -22,13 +24,18 @@ func SetupDefaultConfig() { viper.SetDefault("challenge_response.frequency", 10) viper.SetDefault("challenge_response.num_workers", 5) viper.SetDefault("challenge_response.max_retries", 10) + viper.SetDefault("challenge_response.cleanup_gap", 100000) + viper.SetDefault("rate_limiters.block_limit_daily", 1562500) + viper.SetDefault("rate_limiters.block_limit_request", 500) + viper.SetDefault("rate_limiters.block_limit_monthly", 31250000) + viper.SetDefault("rate_limiters.upload_limit_monthly", 31250000) + viper.SetDefault("rate_limiters.commit_limit_monthly", 30000) viper.SetDefault("healthcheck.frequency", "60s") viper.SetDefault("capacity", -1) viper.SetDefault("read_price", 0.0) viper.SetDefault("write_price", 0.0) - viper.SetDefault("min_lock_demand", 0.0) viper.SetDefault("write_marker_lock_timeout", time.Second*30) @@ -37,6 +44,9 @@ func SetupDefaultConfig() { viper.SetDefault("service_charge", 0.3) viper.SetDefault("update_allocations_interval", time.Duration(-1)) + viper.SetDefault("finalize_allocations_interval", time.Duration(-1)) + + viper.SetDefault("max_dirs_files", 50000) } /*SetupConfig - setup the configuration system */ @@ -58,6 +68,12 @@ func SetupConfig(configPath string) { } Configuration.Config = &config.Configuration + viper.OnConfigChange(func(e fsnotify.Event) { + fmt.Println("Config file changed:", e.Name) + ReadConfig(int(Configuration.DeploymentMode)) + }) + + viper.WatchConfig() } const ( @@ -76,6 +92,7 @@ type Config struct { DBUserName string DBPassword string DBTablesToKeep []string + ArchiveDBPath string OpenConnectionWorkerFreq int64 OpenConnectionWorkerTolerance int64 WMRedeemFreq int64 @@ -87,6 +104,12 @@ type Config struct { ChallengeMaxRetires int TempFilesCleanupFreq int64 TempFilesCleanupNumWorkers int + BlockLimitDaily int64 + BlockLimitRequest int64 + BlockLimitMonthly int64 + UploadLimitMonthly int64 + CommitLimitMonthly int64 + ChallengeCleanupGap int64 HealthCheckWorkerFreq time.Duration @@ -97,7 +120,8 @@ type Config struct { // WriteMarkerLockTimeout lock is released automatically if it is timeout WriteMarkerLockTimeout time.Duration - UpdateAllocationsInterval time.Duration + UpdateAllocationsInterval time.Duration + FinalizeAllocationsInterval time.Duration MaxAllocationDirFiles int @@ -176,11 +200,93 @@ func ValidChain(chain string) error { return ErrSupportedChain } +func ReadConfig(deploymentMode int) { + Configuration.AllocDirLevel = viper.GetIntSlice("storage.alloc_dir_level") + Configuration.FileDirLevel = viper.GetIntSlice("storage.file_dir_level") + Configuration.DeploymentMode = byte(deploymentMode) + Configuration.ChainID = viper.GetString("server_chain.id") + Configuration.SignatureScheme = viper.GetString("server_chain.signature_scheme") + + Configuration.OpenConnectionWorkerFreq = viper.GetInt64("openconnection_cleaner.frequency") + Configuration.OpenConnectionWorkerTolerance = viper.GetInt64("openconnection_cleaner.tolerance") + + Configuration.WMRedeemFreq = viper.GetInt64("writemarker_redeem.frequency") + Configuration.WMRedeemNumWorkers = viper.GetInt("writemarker_redeem.num_workers") + + Configuration.RMRedeemFreq = viper.GetInt64("readmarker_redeem.frequency") + Configuration.RMRedeemNumWorkers = viper.GetInt("readmarker_redeem.num_workers") + + Configuration.HealthCheckWorkerFreq = viper.GetDuration("healthcheck.frequency") + + Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency") + Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers") + Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries") + Configuration.ChallengeCleanupGap = viper.GetInt64("challenge_response.cleanup_gap") + + Configuration.AutomaticUpdate = viper.GetBool("disk_update.automatic_update") + blobberUpdateIntrv := viper.GetDuration("disk_update.blobber_update_interval") + if blobberUpdateIntrv <= 0 { + blobberUpdateIntrv = 5 * time.Minute + } + Configuration.BlobberUpdateInterval = blobberUpdateIntrv + + Configuration.PGUserName = viper.GetString("pg.user") + Configuration.PGPassword = viper.GetString("pg.password") + Configuration.DBHost = viper.GetString("db.host") + Configuration.DBName = viper.GetString("db.name") + Configuration.DBPort = viper.GetString("db.port") + Configuration.DBUserName = viper.GetString("db.user") + Configuration.DBPassword = viper.GetString("db.password") + Configuration.DBTablesToKeep = viper.GetStringSlice("db.keep_tables") + Configuration.ArchiveDBPath = viper.GetString("db.archive_path") + if Configuration.ArchiveDBPath == "" { + Configuration.ArchiveDBPath = "/var/lib/postgresql/hdd" + } + + Configuration.PriceInUSD = viper.GetBool("price_in_usd") + + Configuration.WriteMarkerLockTimeout = viper.GetDuration("write_marker_lock_timeout") + + Configuration.UpdateAllocationsInterval = + viper.GetDuration("update_allocations_interval") + + Configuration.FinalizeAllocationsInterval = + viper.GetDuration("finalize_allocations_interval") + + Configuration.MaxAllocationDirFiles = + viper.GetInt("max_dirs_files") + + Configuration.DelegateWallet = viper.GetString("delegate_wallet") + if w := Configuration.DelegateWallet; len(w) != 64 { + log.Fatal("invalid delegate wallet:", w) + } + + Configuration.MinSubmit = viper.GetInt("min_submit") + if Configuration.MinSubmit < 1 { + Configuration.MinSubmit = 50 + } else if Configuration.MinSubmit > 100 { + Configuration.MinSubmit = 100 + } + Configuration.MinConfirmation = viper.GetInt("min_confirmation") + if Configuration.MinConfirmation < 1 { + Configuration.MinConfirmation = 50 + } else if Configuration.MinConfirmation > 100 { + Configuration.MinConfirmation = 100 + } + + Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") + Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request") + Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") + Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly") + Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly") +} + // StorageSCConfiguration will include all the required sc configs to operate blobber // If any field it required then it can simply be added in this struct and we are // good to go type StorageSCConfiguration struct { - ChallengeCompletionTime time.Duration + ChallengeCompletionTime int64 + MaxFileSize int64 } var StorageSCConfig StorageSCConfiguration diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go index 6def34499..39748dbdd 100644 --- a/code/go/0chain.net/blobbercore/convert/response_creator.go +++ b/code/go/0chain.net/blobbercore/convert/response_creator.go @@ -155,7 +155,7 @@ func CopyObjectResponseCreator(r interface{}) *blobbergrpc.CopyObjectResponse { return nil } - httpResp, _ := r.(*blobberhttp.UploadResult) + httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.CopyObjectResponse{ Filename: httpResp.Filename, Size: httpResp.Size, @@ -171,7 +171,7 @@ func RenameObjectResponseCreator(r interface{}) *blobbergrpc.RenameObjectRespons return nil } - httpResp, _ := r.(*blobberhttp.UploadResult) + httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.RenameObjectResponse{ Filename: httpResp.Filename, Size: httpResp.Size, @@ -209,7 +209,7 @@ func UploadFileResponseCreator(r interface{}) *blobbergrpc.UploadFileResponse { return nil } - httpResp, _ := r.(*blobberhttp.UploadResult) + httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.UploadFileResponse{ Filename: httpResp.Filename, Size: httpResp.Size, diff --git a/code/go/0chain.net/blobbercore/convert/response_handler.go b/code/go/0chain.net/blobbercore/convert/response_handler.go index d7c0fc444..ba4dc171e 100644 --- a/code/go/0chain.net/blobbercore/convert/response_handler.go +++ b/code/go/0chain.net/blobbercore/convert/response_handler.go @@ -117,8 +117,8 @@ func GetCommitMetaTxnHandlerResponse(response *blobbergrpc.CommitMetaTxnResponse return result } -func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *blobberhttp.UploadResult { - return &blobberhttp.UploadResult{ +func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *allocation.UploadResult { + return &allocation.UploadResult{ Filename: copyObjectResponse.Filename, Size: copyObjectResponse.Size, ValidationRoot: copyObjectResponse.ValidationRoot, diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 2bfcde8e3..4e5f4e13f 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -113,7 +113,10 @@ func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error tx.Rollback() return err } - tx.Commit() + err = tx.Commit().Error + if err != nil { + return err + } return nil } func (store *postgresStore) WithTransaction(ctx context.Context, f func(ctx context.Context) error) error { @@ -128,7 +131,10 @@ func (store *postgresStore) WithTransaction(ctx context.Context, f func(ctx cont tx.Rollback() return err } - tx.Commit() + err = tx.Commit().Error + if err != nil { + return err + } return nil } diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index 866ace2d4..a32eddc6f 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -13,8 +13,11 @@ const ( ContextKeyStore ) +type CommitToCahe func(tx *EnhancedDB) + type EnhancedDB struct { - SessionCache map[string]interface{} + SessionCache map[string]interface{} + CommitAllocCache CommitToCahe *gorm.DB } @@ -23,6 +26,16 @@ func EnhanceDB(db *gorm.DB) *EnhancedDB { return &EnhancedDB{DB: db, SessionCache: cache} } +func (edb *EnhancedDB) Commit() *gorm.DB { + db := edb.DB.Commit() + if db.Error == nil { + if edb.CommitAllocCache != nil { + edb.CommitAllocCache(edb) + } + } + return db +} + type Store interface { // GetDB get raw gorm db diff --git a/code/go/0chain.net/blobbercore/filestore/state.go b/code/go/0chain.net/blobbercore/filestore/state.go index b07b8c204..b098a34c8 100644 --- a/code/go/0chain.net/blobbercore/filestore/state.go +++ b/code/go/0chain.net/blobbercore/filestore/state.go @@ -90,7 +90,6 @@ func (fs *FileStore) initMap() error { }).Error wg.Wait() - db.Commit() return err }) return err diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 8d96902cc..e12f2a44a 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -60,7 +60,11 @@ const ( func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) { tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileData.FilePathHash, conID) - var initialSize int64 + var ( + initialSize int64 + nodeSize int64 + offset int64 + ) finfo, err := os.Stat(tempFilePath) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, common.NewError("file_stat_error", err.Error()) @@ -68,18 +72,20 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i if finfo != nil { initialSize = finfo.Size() } - + if !fileData.IsThumbnail { + nodeSize = getNodesSize(fileData.Size, util.MaxMerkleLeavesSize) + offset = fileData.UploadOffset + nodeSize + FMTSize + } if err = createDirs(filepath.Dir(tempFilePath)); err != nil { return nil, common.NewError("dir_creation_error", err.Error()) } - f, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return nil, common.NewError("file_open_error", err.Error()) } defer f.Close() - _, err = f.Seek(fileData.UploadOffset, io.SeekStart) + _, err = f.Seek(offset, io.SeekStart) if err != nil { return nil, common.NewError("file_seek_error", err.Error()) } @@ -99,11 +105,10 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i currentSize := finfo.Size() if currentSize > initialSize { // Is chunk new or rewritten if !fileData.IsThumbnail { - _, err = f.Seek(fileData.UploadOffset, io.SeekStart) + _, err = f.Seek(offset, io.SeekStart) if err != nil { return nil, common.NewError("file_seek_error", err.Error()) } - _, err = io.CopyBuffer(fileData.Hasher, f, buf) if err != nil { return nil, common.NewError("file_read_error", err.Error()) @@ -112,11 +117,10 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i fileRef.ChunkUploaded = true fs.updateAllocTempFileSize(allocID, currentSize-initialSize) } - + fmt.Println("writtenSize", writtenSize, offset) fileRef.Size = writtenSize fileRef.Name = fileData.Name fileRef.Path = fileData.Path - return fileRef, nil } @@ -187,18 +191,11 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) return false, common.NewError("blob_object_precommit_dir_creation_error", err.Error()) } - f, err := os.Create(preCommitPath) - if err != nil { - return false, common.NewError("file_create_error", err.Error()) - } - - r, err := os.Open(tempFilePath) + r, err := os.OpenFile(tempFilePath, os.O_RDWR, 0644) if err != nil { return false, err } - defer f.Close() - defer func() { r.Close() if err != nil { @@ -231,7 +228,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) return false, err } - _, err = io.Copy(f, r) + err = os.Rename(tempFilePath, preCommitPath) if err != nil { return false, err } @@ -252,20 +249,14 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) if err != nil { return false, common.NewError("stat_error", err.Error()) } - - fileSize := rStat.Size() - bufSize := BufferSize - if fileSize < BufferSize { - bufSize = int(fileSize) - } - buffer := make([]byte, bufSize) - - fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(f) + nodeSie := getNodesSize(fileData.Size, util.MaxMerkleLeavesSize) + fileSize := rStat.Size() - nodeSie - FMTSize + fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(r) if err != nil { return false, common.NewError("fmt_hash_calculation_error", err.Error()) } - validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(f) + validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(r) if err != nil { return false, common.NewError("validation_hash_calculation_error", err.Error()) } @@ -281,7 +272,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) "calculated validation root does not match with client's validation root") } - _, err = io.CopyBuffer(f, r, buffer) + err = os.Rename(tempFilePath, preCommitPath) if err != nil { return false, common.NewError("write_error", err.Error()) } @@ -537,6 +528,7 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes dataSize: readBlockIn.FileSize, } + logging.Logger.Debug("calling GetMerkleProofOfMultipleIndexes", zap.Any("readBlockIn", readBlockIn), zap.Any("vmp", vmp)) nodes, indexes, err := vp.GetMerkleProofOfMultipleIndexes(file, nodesSize, startBlock, endBlock) if err != nil { return nil, common.NewError("get_merkle_proof_error", err.Error()) diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 9d4a614e7..8a038458c 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -23,6 +23,7 @@ type FileInputData struct { IsFinal bool IsThumbnail bool FilePathHash string + Size int64 Hasher *CommitHasher } diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index bcef5f2ec..afe95f474 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -162,7 +162,7 @@ func setupMockForFileManagerInit(mock sqlmock.Sqlmock, ip initParams) { sqlmock.NewRows([]string{"file_size"}).AddRow(ip.usedSize), ) - mock.ExpectClose() + mock.ExpectCommit() } @@ -258,6 +258,7 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { ChunkSize: 64 * KB, FilePathHash: pathHash, Hasher: hasher, + Size: int64(size), } f, err := os.Open(fPath) @@ -274,8 +275,8 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { finfo, err := f.Stat() require.Nil(t, err) - - require.Equal(t, finfo.Size(), tF.Size()) + nodeSize := getNodesSize(int64(finfo.Size()), util.MaxMerkleLeavesSize) + require.Equal(t, finfo.Size(), tF.Size()-nodeSize-FMTSize) if !test.shouldCommit { return @@ -305,7 +306,7 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { require.Nil(t, err) check_file, err := os.Stat(finalPath) require.Nil(t, err) - require.True(t, check_file.Size() > tF.Size()) + require.True(t, check_file.Size() == tF.Size()) } }) } @@ -342,6 +343,7 @@ func TestDeletePreCommitDir(t *testing.T) { ChunkSize: 64 * KB, FilePathHash: pathHash, Hasher: hasher, + Size: int64(size), } // checkc if file to be uploaded exists f, err := os.Open(fPath) @@ -356,8 +358,8 @@ func TestDeletePreCommitDir(t *testing.T) { tempFilePath := fs.getTempPathForFile(allocID, fileName, pathHash, connID) tF, err := os.Stat(tempFilePath) require.Nil(t, err) - - require.Equal(t, int64(size), tF.Size()) + nodeSize := getNodesSize(int64(size), util.MaxMerkleLeavesSize) + require.Equal(t, int64(size), tF.Size()-nodeSize-FMTSize) // Commit file to pre-commit location success, err := fs.CommitWrite(allocID, connID, fid) @@ -442,6 +444,7 @@ func TestStorageUploadUpdate(t *testing.T) { ChunkSize: 64 * KB, FilePathHash: pathHash, Hasher: hasher, + Size: int64(size), } // checkc if file to be uploaded exists f, err := os.Open(fPath) @@ -456,8 +459,8 @@ func TestStorageUploadUpdate(t *testing.T) { tempFilePath := fs.getTempPathForFile(allocID, fileName, pathHash, connID) tF, err := os.Stat(tempFilePath) require.Nil(t, err) - - require.Equal(t, int64(size), tF.Size()) + nodeSize := getNodesSize(int64(size), util.MaxMerkleLeavesSize) + require.Equal(t, int64(size), tF.Size()-nodeSize-FMTSize) // Commit file to pre-commit location success, err := fs.CommitWrite(allocID, connID, fid) @@ -798,7 +801,7 @@ func TestGetMerkleTree(t *testing.T) { rootHash, _ := hex.DecodeString(fixedMerkleRoot) fmp := &util.FixedMerklePath{ - LeafHash: encryption.RawHash(challengeProof.Data), + LeafHash: encryption.ShaHash(challengeProof.Data), RootHash: rootHash, Nodes: challengeProof.Proof, LeafInd: test.blockOffset, diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index b44145707..de8eae335 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -12,7 +12,7 @@ import ( "sync" "github.com/0chain/gosdk/core/util" - "golang.org/x/crypto/sha3" + "github.com/minio/sha256-simd" ) const ( @@ -66,7 +66,7 @@ func (ft *fixedMerkleTree) CalculateRootAndStoreNodes(f io.Writer) (merkleRoot [ buffer := make([]byte, FMTSize) var bufLen int - h := sha3.New256() + h := sha256.New() for i := 0; i < util.FixedMTDepth; i++ { if len(nodes) == 1 { @@ -83,7 +83,7 @@ func (ft *fixedMerkleTree) CalculateRootAndStoreNodes(f io.Writer) (merkleRoot [ bufLen += copy(buffer[bufLen:bufLen+HashSize], nodes[j]) bufLen += copy(buffer[bufLen:bufLen+HashSize], nodes[j+1]) - h.Write(buffer[prevBufLen:bufLen]) + _, _ = h.Write(buffer[prevBufLen:bufLen]) newNodes[nodeInd] = h.Sum(nil) nodeInd++ } @@ -228,7 +228,7 @@ func (v *validationTree) CalculateRootAndStoreNodes(f io.WriteSeeker) (merkleRoo nodes := make([][]byte, len(v.GetLeaves())) copy(nodes, v.GetLeaves()) - h := sha3.New256() + h := sha256.New() depth := v.CalculateDepth() s := getNodesSize(v.GetDataSize(), util.MaxMerkleLeavesSize) @@ -246,7 +246,7 @@ func (v *validationTree) CalculateRootAndStoreNodes(f io.WriteSeeker) (merkleRoo bufInd += copy(buffer[bufInd:], nodes[j]) bufInd += copy(buffer[bufInd:], nodes[j+1]) - h.Write(buffer[prevBufInd:bufInd]) + _, _ = h.Write(buffer[prevBufInd:bufInd]) newNodes = append(newNodes, h.Sum(nil)) } } else { @@ -256,13 +256,13 @@ func (v *validationTree) CalculateRootAndStoreNodes(f io.WriteSeeker) (merkleRoo bufInd += copy(buffer[bufInd:], nodes[j]) bufInd += copy(buffer[bufInd:], nodes[j+1]) - h.Write(buffer[prevBufInd:bufInd]) + _, _ = h.Write(buffer[prevBufInd:bufInd]) newNodes = append(newNodes, h.Sum(nil)) } h.Reset() prevBufInd := bufInd bufInd += copy(buffer[bufInd:], nodes[len(nodes)-1]) - h.Write(buffer[prevBufInd:bufInd]) + _, _ = h.Write(buffer[prevBufInd:bufInd]) newNodes = append(newNodes, h.Sum(nil)) } @@ -288,7 +288,7 @@ type validationTreeProof struct { // If endInd - startInd is whole file then no proof is required at all. // startInd and endInd is taken as closed interval. So to get proof for data at index 0 both startInd // and endInd would be 0. -func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( +func (v *validationTreeProof) getMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( [][][]byte, [][]int, error) { if startInd < 0 || endInd < 0 { diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go index 6e8612e0f..4d12fcfff 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/0chain/gosdk/core/util" - "golang.org/x/crypto/sha3" + "github.com/minio/sha256-simd" ) func BenchmarkFixedMerkleProofFor10MB(b *testing.B) { @@ -81,8 +81,8 @@ func runFixedMPBench(b *testing.B, size int64, filename string) { b.Fatalf("error: %v", err) } - h := sha3.New256() - h.Write(proofByte) + h := sha256.New() + _, _ = h.Write(proofByte) fp := util.FixedMerklePath{ LeafHash: h.Sum(nil), diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go new file mode 100644 index 000000000..f5a51fc82 --- /dev/null +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go @@ -0,0 +1,59 @@ +//go:build integration_tests +// +build integration_tests + +package filestore + +import ( + "io" + + crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" +) + +func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( + [][][]byte, [][]int, error) { + nodes, indexes, err := v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) + if err != nil { + return nodes, indexes, err + } + + state := crpc.Client().State() + if state == nil { + return nodes, indexes, err + } + + logging.Logger.Debug("[conductor] just after getMerkleProofOfMultipleIndexes", + zap.Bool("miss_up_download", state.MissUpDownload), + zap.Int("start_ind", startInd), + zap.Int("end_ind", endInd), + zap.Int64("nodes_size", nodesSize), + zap.Int("output_nodes_size", len(nodes)), + zap.Int("output_indexes_size", len(indexes)), + zap.Any("nodes", nodes), + zap.Any("indexes", indexes), + ) + + if state.MissUpDownload { + logging.Logger.Debug("miss up/download", + zap.Bool("miss_up_download", state.MissUpDownload), + zap.Int("start_ind", startInd), + zap.Int("end_ind", endInd), + zap.Int64("nodes_size", nodesSize), + zap.Int("output_nodes_size", len(nodes)), + zap.Int("output_indexes_size", len(indexes)), + zap.Any("nodes", nodes), + zap.Any("indexes", indexes), + ) + + for i := range nodes { + nodes[i][0] = []byte("wrong data") + } + + for i := range indexes { + indexes[i][0] = 0 + } + } + + return nodes, indexes, err +} \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go new file mode 100644 index 000000000..d0dc7a5a2 --- /dev/null +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_main.go @@ -0,0 +1,11 @@ +//go:build !integration_tests +// +build !integration_tests + +package filestore + +import "io" + +func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( + [][][]byte, [][]int, error) { + return v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) +} diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go index 02b3bb33c..f3097474a 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go @@ -162,7 +162,7 @@ func TestFixedMerkleTreeProof(t *testing.T) { proofByte, err := fm.GetLeafContent(r) require.NoError(t, err) - leafHash := encryption.RawHash(proofByte) + leafHash := encryption.ShaHash(proofByte) ftp := util.FixedMerklePath{ LeafHash: leafHash, RootHash: merkleRoot, @@ -231,7 +231,7 @@ func TestValidationTreeWrite(t *testing.T) { n, err = r.Read(b) require.NoError(t, err) require.EqualValues(t, size, n) - leaves[0] = encryption.RawHash(b) + leaves[0] = encryption.ShaHash(b) } else { nodes := make([]byte, totalLeaves*HashSize) n, err = r.Read(nodes) diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go new file mode 100644 index 000000000..b4d314dd5 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/client_quota.go @@ -0,0 +1,153 @@ +package handler + +import ( + "context" + "sync" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" +) + +var ( + clientMap = make(map[string]*ClientStats) + mpLock sync.RWMutex + blackListMap = make(map[string]bool) + blMap sync.RWMutex +) + +const ( + Period = 60 * 60 * 24 * 30 // 30 days +) + +type ClientStats struct { + ClientID string `gorm:"column:client_id;size:64;primaryKey" json:"client_id"` + CreatedAt common.Timestamp `gorm:"created_at;primaryKey" json:"created"` + TotalUpload uint64 `gorm:"column:total_upload" json:"total_upload"` + TotalDownload uint64 `gorm:"column:total_download" json:"total_download"` + TotalWM uint64 `gorm:"column:total_write_marker" json:"total_write_marker"` +} + +func (ClientStats) TableName() string { + return "client_stats" +} + +func (cs *ClientStats) BeforeCreate() error { + cs.CreatedAt = common.Now() + return nil +} + +func GetUploadedData(clientID string) uint64 { + mpLock.RLock() + defer mpLock.RUnlock() + cs := clientMap[clientID] + if cs == nil { + return 0 + } + return cs.TotalUpload +} + +func AddUploadedData(clientID string, data int64) { + mpLock.Lock() + defer mpLock.Unlock() + cs := clientMap[clientID] + if cs == nil { + cs = &ClientStats{ClientID: clientID} + clientMap[clientID] = cs + } + cs.TotalUpload += uint64(data) +} + +func GetDownloadedData(clientID string) uint64 { + mpLock.RLock() + defer mpLock.RUnlock() + cs := clientMap[clientID] + return cs.TotalDownload +} + +func AddDownloadedData(clientID string, data int64) { + mpLock.Lock() + defer mpLock.Unlock() + cs := clientMap[clientID] + if cs == nil { + cs = &ClientStats{ClientID: clientID} + clientMap[clientID] = cs + } + cs.TotalDownload += uint64(data) +} + +func GetWriteMarkerCount(clientID string) uint64 { + mpLock.RLock() + defer mpLock.RUnlock() + cs := clientMap[clientID] + if cs == nil { + return 0 + } + return cs.TotalWM +} + +func AddWriteMarkerCount(clientID string, count int64) { + mpLock.Lock() + defer mpLock.Unlock() + cs := clientMap[clientID] + if cs == nil { + cs = &ClientStats{ClientID: clientID} + clientMap[clientID] = cs + } + cs.TotalWM += uint64(count) +} + +func CheckBlacklist(clientID string) bool { + blMap.RLock() + defer blMap.RUnlock() + _, ok := blackListMap[clientID] + return ok +} + +func saveClientStats() { + dbStats := make([]*ClientStats, 0, len(clientMap)) + mpLock.Lock() + for _, cs := range clientMap { + dbStats = append(dbStats, cs) + delete(clientMap, cs.ClientID) + } + mpLock.Unlock() + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + if len(dbStats) > 0 { + tx := datastore.GetStore().GetTransaction(ctx) + return tx.Create(dbStats).Error + } + return nil + }) + var blackList []string + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + tx := datastore.GetStore().GetTransaction(ctx) + err := tx.Raw("SELECT client_id as blackList from (SELECT client_id,sum(total_upload) as upload,sum(total_download) as download, sum(total_write_marker) as writemarker from client_stats where created_at > ? group by client_id) as stats where stats.upload > ? or stats.download > ? or stats.writemarker > ?", common.Now()-Period, config.Configuration.UploadLimitMonthly, config.Configuration.BlockLimitMonthly, config.Configuration.CommitLimitMonthly).Scan(&blackList).Error + return err + }) + if err == nil { + blMap.Lock() + blackListMap = make(map[string]bool) + for _, clientID := range blackList { + blackListMap[clientID] = true + } + blMap.Unlock() + } +} + +func startBlackListWorker(ctx context.Context) { + BlackListWorkerTime := 6 * time.Hour + if config.Development() { + BlackListWorkerTime = 10 * time.Second + } + + for { + select { + case <-ctx.Done(): + return + case <-time.After(BlackListWorkerTime): + saveClientStats() + } + } +} diff --git a/code/go/0chain.net/blobbercore/handler/download_quota.go b/code/go/0chain.net/blobbercore/handler/download_quota.go index 8d731cc5f..4f14b4197 100644 --- a/code/go/0chain.net/blobbercore/handler/download_quota.go +++ b/code/go/0chain.net/blobbercore/handler/download_quota.go @@ -1,8 +1,10 @@ package handler import ( + "context" "fmt" "sync" + "time" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) @@ -17,8 +19,37 @@ type QuotaManager struct { mux sync.RWMutex } -var quotaManagerInstance *QuotaManager -var quotaManagerOnce sync.Once +var ( + quotaManagerInstance *QuotaManager + quotaManagerOnce sync.Once + downloadLimit = make(map[string]int64) + downloadLock sync.RWMutex +) + +func addDailyBlocks(key string, numBlocks int64) { + downloadLock.Lock() + defer downloadLock.Unlock() + downloadLimit[key] += numBlocks +} + +func getDailyBlocks(key string) int64 { + downloadLock.RLock() + defer downloadLock.RUnlock() + return downloadLimit[key] +} + +func startDownloadLimitCleanup(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(24 * time.Hour): + downloadLock.Lock() + downloadLimit = make(map[string]int64) + downloadLock.Unlock() + } + } +} func getQuotaManager() *QuotaManager { quotaManagerOnce.Do(func() { diff --git a/code/go/0chain.net/blobbercore/handler/file_command.go b/code/go/0chain.net/blobbercore/handler/file_command.go index 2035f98b0..078d4241c 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command.go +++ b/code/go/0chain.net/blobbercore/handler/file_command.go @@ -6,35 +6,15 @@ import ( "net/http" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // FileCommand execute command for a file operation -type FileCommand interface { - - // GetExistingFileRef get file ref if it exists - GetExistingFileRef() *reference.Ref - - GetPath() string - - // IsValidated validate request, and try build ChangeProcesser instance - IsValidated(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error - - // ProcessContent flush file to FileStorage - ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) - - // ProcessThumbnail flush thumbnail file to FileStorage if it has. - ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error - - // UpdateChange update AllocationChangeProcessor. It will be president in db for committing transcation - UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error -} // createFileCommand create file command for UPLOAD,UPDATE and DELETE -func createFileCommand(req *http.Request) FileCommand { +func createFileCommand(req *http.Request) allocation.FileCommand { switch req.Method { case http.MethodPost: return &UploadFileCommand{} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 8c51bbdb3..7b6c49aa3 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -9,9 +9,9 @@ import ( "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) // DeleteFileCommand command for deleting file @@ -20,6 +20,7 @@ type DeleteFileCommand struct { changeProcessor *allocation.DeleteFileChange allocationChange *allocation.AllocationChange path string + connectionID string } func (cmd *DeleteFileCommand) GetExistingFileRef() *reference.Ref { @@ -43,15 +44,28 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request cmd.path = path + connectionID, ok := common.GetField(req, "connection_id") + if !ok { + return common.NewError("invalid_parameters", "Invalid connection id passed") + } + cmd.connectionID = connectionID var err error - cmd.existingFileRef, err = reference.GetLimitedRefFieldsByPath(ctx, allocationObj.ID, path, []string{"path", "name", "size", "hash", "fixed_merkle_root"}) + pathHash := encryption.Hash(path) + err = allocation.GetError(connectionID, pathHash) + if err != nil { + return err + } + lookUpHash := reference.GetReferenceLookup(allocationObj.ID, path) + cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "size", "hash", "fixed_merkle_root"}) if err != nil { if errors.Is(gorm.ErrRecordNotFound, err) { return common.ErrFileWasDeleted } return common.NewError("bad_db_operation", err.Error()) } - return nil + allocation.CreateConnectionChange(connectionID, pathHash, allocationObj) + + return allocation.SetFinalized(connectionID, pathHash, cmd) } // UpdateChange add DeleteFileChange in db @@ -62,32 +76,36 @@ func (cmd *DeleteFileCommand) UpdateChange(ctx context.Context, connectionObj *a } // ProcessContent flush file to FileStorage -func (cmd *DeleteFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { +func (cmd *DeleteFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) { deleteSize := cmd.existingFileRef.Size - - cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, Name: cmd.existingFileRef.Name, + connectionID := cmd.connectionID + cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID, + AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name, Hash: cmd.existingFileRef.Hash, Path: cmd.existingFileRef.Path, Size: deleteSize} - result := blobberhttp.UploadResult{} + result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name result.ValidationRoot = cmd.existingFileRef.ValidationRoot result.FixedMerkleRoot = cmd.existingFileRef.FixedMerkleRoot result.Size = cmd.existingFileRef.Size + result.IsFinal = true cmd.allocationChange = &allocation.AllocationChange{} - cmd.allocationChange.ConnectionID = connectionObj.ID + cmd.allocationChange.ConnectionID = connectionID cmd.allocationChange.Size = 0 - deleteSize cmd.allocationChange.Operation = constants.FileOperationDelete - connectionObj.Size += cmd.allocationChange.Size - allocation.UpdateConnectionObjSize(connectionObj.ID, cmd.allocationChange.Size) + allocation.UpdateConnectionObjSize(connectionID, cmd.allocationChange.Size) return result, nil } // ProcessThumbnail no thumbnail should be processed for delete. A deffered delete command has been added on ProcessContent -func (cmd *DeleteFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { +func (cmd *DeleteFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error { //DO NOTHING return nil } + +func (cmd *DeleteFileCommand) GetNumBlocks() int64 { + return 0 +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 32b58b822..07e98d030 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -4,10 +4,13 @@ import ( "context" "encoding/json" "fmt" + "math" + "mime/multipart" "net/http" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -27,6 +30,9 @@ type UpdateFileCommand struct { existingFileRef *reference.Ref fileChanger *allocation.UpdateFileChanger allocationChange *allocation.AllocationChange + contentFile multipart.File + thumbFile multipart.File + thumbHeader *multipart.FileHeader } func (cmd *UpdateFileCommand) GetExistingFileRef() *reference.Ref { @@ -42,6 +48,10 @@ func (cmd *UpdateFileCommand) GetPath() string { // IsValidated validate request. func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error { + if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { + return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + uploadMetaString := req.FormValue(UploadMeta) if uploadMetaString == "" { @@ -54,64 +64,91 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_parameters", "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } - logging.Logger.Info("UpdateFileCommand", zap.Any("allocation_id", allocationObj.ID), zap.Any("validation_rooot", cmd.fileChanger.ValidationRoot), zap.Any("thumb_hash", cmd.fileChanger.ThumbnailHash)) + + if cmd.fileChanger.Size > config.StorageSCConfig.MaxFileSize { + return common.NewError("max_file_size", + fmt.Sprintf("file size %d should not be greater than %d", cmd.fileChanger.Size, config.StorageSCConfig.MaxFileSize)) + } + + if cmd.fileChanger.ConnectionID == "" { + return common.NewError("invalid_connection", "Invalid connection id") + } + + cmd.fileChanger.PathHash = encryption.Hash(cmd.fileChanger.Path) if cmd.fileChanger.ChunkSize <= 0 { cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE } - cmd.existingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path) - if cmd.existingFileRef == nil { - return common.NewError("invalid_file_update", "File at path does not exist for update") + err = allocation.GetError(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + if err != nil { + return err } - if allocationObj.OwnerID != clientID && - allocationObj.RepairerID != clientID { - return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + // Check if ref exists at start of update or get existing ref + if cmd.fileChanger.UploadOffset == 0 { + logging.Logger.Info("UpdateFile ref exists check") + cmd.existingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path) + if cmd.existingFileRef == nil { + return common.NewError("invalid_file_update", "File at path does not exist for update") + } + logging.Logger.Info("UpdateFile ref exists check done", zap.Any("ref", cmd.existingFileRef)) + allocation.CreateConnectionChange(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, allocationObj) + err = allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd.existingFileRef) + if err != nil { + return common.NewError("invalid_file_update", "Error saving existing ref") + } + } else { + cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + if cmd.existingFileRef == nil { + return common.NewError("invalid_file_update", "Existing file reference is nil") + } } - _, thumbHeader, _ := req.FormFile(UploadThumbnailFile) + thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { if thumbHeader.Size > MaxThumbnailSize { return common.NewError("max_thumbnail_size", fmt.Sprintf("thumbnail size %d should not be greater than %d", thumbHeader.Size, MaxThumbnailSize)) } + cmd.thumbFile = thumbFile + cmd.thumbHeader = thumbHeader } - return nil + origfile, _, err := req.FormFile(UploadFile) + if err != nil { + return common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + cmd.contentFile = origfile + if cmd.fileChanger.IsFinal { + return allocation.SetFinalized(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd) + } + return allocation.SendCommand(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd) } // ProcessContent flush file to FileStorage -func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - result := blobberhttp.UploadResult{} +func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) { + result := allocation.UploadResult{} result.Filename = cmd.fileChanger.Filename - - isFinal := cmd.fileChanger.IsFinal - cmd.fileChanger.IsFinal = false - cmd.reloadChange(connectionObj) + defer cmd.contentFile.Close() if cmd.fileChanger.IsFinal { - return result, nil + cmd.reloadChange() } - cmd.fileChanger.IsFinal = isFinal - - origfile, _, err := req.FormFile(UploadFile) - if err != nil { - return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) - } - defer origfile.Close() if cmd.fileChanger.Size == 0 { return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") } var hasher *filestore.CommitHasher - filePathHash := encryption.Hash(cmd.fileChanger.Path) + filePathHash := cmd.fileChanger.PathHash + connID := cmd.fileChanger.ConnectionID if cmd.fileChanger.UploadOffset == 0 { + result.UpdateChange = true hasher = filestore.GetNewCommitHasher(cmd.fileChanger.Size) - allocation.UpdateConnectionObjWithHasher(connectionObj.ID, filePathHash, hasher) + allocation.UpdateConnectionObjWithHasher(connID, filePathHash, hasher) } else { - hasher = allocation.GetHasher(connectionObj.ID, filePathHash) + hasher = allocation.GetHasher(connID, filePathHash) if hasher == nil { return result, common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for upload.") } @@ -124,8 +161,9 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ IsFinal: cmd.fileChanger.IsFinal, FilePathHash: filePathHash, Hasher: hasher, + Size: cmd.fileChanger.Size, } - fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, fileInputData, origfile) + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connID, fileInputData, cmd.contentFile) if err != nil { return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } @@ -135,17 +173,18 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ if err != nil { return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } + result.IsFinal = true } result.ValidationRoot = fileOutputData.ValidationRoot result.FixedMerkleRoot = fileOutputData.FixedMerkleRoot result.Size = fileOutputData.Size - allocationSize := connectionObj.Size + allocationSize := allocation.GetConnectionObjSize(connID) if fileOutputData.ChunkUploaded { allocationSize += fileOutputData.Size - allocation.UpdateConnectionObjSize(connectionObj.ID, fileOutputData.Size) + allocation.UpdateConnectionObjSize(connID, fileOutputData.Size) } if allocationObj.BlobberSizeUsed+(allocationSize-cmd.existingFileRef.Size) > allocationObj.BlobberSize { @@ -153,63 +192,45 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ } cmd.fileChanger.AllocationID = allocationObj.ID - // cmd.fileChanger.Size += fileOutputData.Size cmd.allocationChange = &allocation.AllocationChange{} - cmd.allocationChange.ConnectionID = connectionObj.ID + cmd.allocationChange.ConnectionID = connID cmd.allocationChange.Size = cmd.fileChanger.Size - cmd.existingFileRef.Size cmd.allocationChange.Operation = sdkConst.FileOperationUpdate if cmd.fileChanger.IsFinal { - connectionObj.Size = allocationSize - cmd.existingFileRef.Size - allocation.UpdateConnectionObjSize(connectionObj.ID, -cmd.existingFileRef.Size) - } else { - connectionObj.Size = allocationSize + allocation.UpdateConnectionObjSize(connID, -cmd.existingFileRef.Size) } return result, nil } // ProcessThumbnail flush thumbnail file to FileStorage if it has. -func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - thumbfile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) +func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error { + connectionID := cmd.fileChanger.ConnectionID + if cmd.thumbHeader != nil { + defer cmd.thumbFile.Close() - if thumbHeader != nil { - defer thumbfile.Close() - - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: encryption.Hash(cmd.fileChanger.Path)} - thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, thumbInputData, thumbfile) + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) } cmd.fileChanger.ThumbnailSize = thumbOutputData.Size cmd.fileChanger.ThumbnailFilename = thumbInputData.Name + err = allocation.SaveFileChanger(connectionID, &cmd.fileChanger.BaseFileChanger) + return err } - - return nil + return common.ErrNoThumbnail } -func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { - for _, c := range connectionObj.Changes { - filePath, _ := c.GetOrParseAffectedFilePath() - if c.Operation != sdkConst.FileOperationUpdate || cmd.fileChanger.Path != filePath { - continue - } - - dbFileChanger := &allocation.UpdateFileChanger{} - - err := dbFileChanger.Unmarshal(c.Input) - if err != nil { - logging.Logger.Error("reloadChange", zap.Error(err)) - } - - // reload uploaded size from db, it was chunk size from client - cmd.fileChanger.ThumbnailFilename = dbFileChanger.ThumbnailFilename - cmd.fileChanger.ThumbnailSize = dbFileChanger.ThumbnailSize - cmd.fileChanger.ThumbnailHash = dbFileChanger.ThumbnailHash - cmd.fileChanger.IsFinal = dbFileChanger.IsFinal - return +func (cmd *UpdateFileCommand) reloadChange() { + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + if changer != nil { + cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize + cmd.fileChanger.ThumbnailHash = changer.ThumbnailHash } } @@ -239,3 +260,10 @@ func (cmd *UpdateFileCommand) UpdateChange(ctx context.Context, connectionObj *a return connectionObj.Save(ctx) } + +func (cmd *UpdateFileCommand) GetNumBlocks() int64 { + if cmd.fileChanger.IsFinal { + return int64(math.Ceil(float64(cmd.fileChanger.Size*1.0) / float64(cmd.fileChanger.ChunkSize))) + } + return 0 +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 76419f51f..9176979ea 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -4,20 +4,23 @@ import ( "context" "encoding/json" "fmt" + "math" + "mime/multipart" "net/http" "path/filepath" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" - "go.uber.org/zap" ) const ( @@ -31,6 +34,9 @@ const ( type UploadFileCommand struct { allocationChange *allocation.AllocationChange fileChanger *allocation.UploadFileChanger + contentFile multipart.File + thumbFile multipart.File + thumbHeader *multipart.FileHeader } func (cmd *UploadFileCommand) GetExistingFileRef() *reference.Ref { @@ -58,6 +64,11 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } + if fileChanger.Size > config.StorageSCConfig.MaxFileSize { + return common.NewError("max_file_size", + fmt.Sprintf("file size %d should not be greater than %d", fileChanger.Size, config.StorageSCConfig.MaxFileSize)) + } + if fileChanger.Path == "/" { return common.NewError("invalid_path", "Invalid path. Cannot upload to root directory") } @@ -66,63 +77,77 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path)) } - isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path) + if fileChanger.ConnectionID == "" { + return common.NewError("invalid_connection", "Invalid connection id") + } + fileChanger.PathHash = encryption.Hash(fileChanger.Path) + + err = allocation.GetError(fileChanger.ConnectionID, fileChanger.PathHash) if err != nil { - logging.Logger.Error(err.Error()) - return common.NewError("database_error", "Got db error while getting ref") + return err } - if isExist { - msg := fmt.Sprintf("File at path :%s: already exists", fileChanger.Path) - return common.NewError("duplicate_file", msg) + if fileChanger.UploadOffset == 0 { + isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path) + + if err != nil { + logging.Logger.Error(err.Error()) + return common.NewError("database_error", "Got db error while getting ref") + } + + if isExist { + msg := fmt.Sprintf("File at path :%s: already exists", fileChanger.Path) + return common.NewError("duplicate_file", msg) + } + allocation.CreateConnectionChange(fileChanger.ConnectionID, fileChanger.PathHash, allocationObj) } - _, thumbHeader, _ := req.FormFile(UploadThumbnailFile) + thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { if thumbHeader.Size > MaxThumbnailSize { return common.NewError("max_thumbnail_size", fmt.Sprintf("thumbnail size %d should not be greater than %d", thumbHeader.Size, MaxThumbnailSize)) } + cmd.thumbFile = thumbFile + cmd.thumbHeader = thumbHeader } if fileChanger.ChunkSize <= 0 { fileChanger.ChunkSize = fileref.CHUNK_SIZE } + origfile, _, err := req.FormFile(UploadFile) + if err != nil { + return common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) + } + cmd.contentFile = origfile cmd.fileChanger = fileChanger - return nil + logging.Logger.Info("UploadFileCommand.IsValidated") + if fileChanger.IsFinal { + return allocation.SetFinalized(fileChanger.ConnectionID, fileChanger.PathHash, cmd) + } + return allocation.SendCommand(fileChanger.ConnectionID, fileChanger.PathHash, cmd) } // ProcessContent flush file to FileStorage -func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) { - result := blobberhttp.UploadResult{} - - origfile, _, err := req.FormFile(UploadFile) - if err != nil { - return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) - } - defer origfile.Close() - isFinal := cmd.fileChanger.IsFinal - cmd.fileChanger.IsFinal = false - cmd.reloadChange(connectionObj) +func (cmd *UploadFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) { + result := allocation.UploadResult{} + defer cmd.contentFile.Close() if cmd.fileChanger.IsFinal { - result.Filename = cmd.fileChanger.Filename - return result, nil + cmd.reloadChange() } - cmd.fileChanger.IsFinal = isFinal - + connectionID := cmd.fileChanger.ConnectionID var hasher *filestore.CommitHasher - filePathHash := encryption.Hash(cmd.fileChanger.Path) if cmd.fileChanger.Size == 0 { return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") } - if cmd.fileChanger.UploadOffset == 0 { + result.UpdateChange = true hasher = filestore.GetNewCommitHasher(cmd.fileChanger.Size) - allocation.UpdateConnectionObjWithHasher(connectionObj.ID, filePathHash, hasher) + allocation.UpdateConnectionObjWithHasher(connectionID, cmd.fileChanger.PathHash, hasher) } else { - hasher = allocation.GetHasher(connectionObj.ID, filePathHash) + hasher = allocation.GetHasher(connectionID, cmd.fileChanger.PathHash) if hasher == nil { return result, common.NewError("invalid_parameters", "Error getting hasher for upload.") } @@ -135,11 +160,13 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ ChunkSize: cmd.fileChanger.ChunkSize, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: filePathHash, + FilePathHash: cmd.fileChanger.PathHash, Hasher: hasher, + Size: cmd.fileChanger.Size, } - fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, fileInputData, origfile) + fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, fileInputData, cmd.contentFile) if err != nil { + logging.Logger.Error("UploadFileCommand.ProcessContent", zap.Error(err)) return result, common.NewError("upload_error", "Failed to write file. "+err.Error()) } @@ -148,18 +175,19 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ if err != nil { return result, common.NewError("upload_error", "Failed to finalize the hasher. "+err.Error()) } + result.IsFinal = true } result.Filename = cmd.fileChanger.Filename result.ValidationRoot = fileOutputData.ValidationRoot result.Size = fileOutputData.Size - allocationSize := connectionObj.Size + allocationSize := allocation.GetConnectionObjSize(connectionID) // only update connection size when the chunk is uploaded. if fileOutputData.ChunkUploaded { allocationSize += fileOutputData.Size - allocation.UpdateConnectionObjSize(connectionObj.ID, fileOutputData.Size) + allocation.UpdateConnectionObjSize(connectionID, fileOutputData.Size) } if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { @@ -167,57 +195,40 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ } cmd.fileChanger.AllocationID = allocationObj.ID - // cmd.fileChanger.Size += fileOutputData.Size cmd.allocationChange = &allocation.AllocationChange{} - cmd.allocationChange.ConnectionID = connectionObj.ID + cmd.allocationChange.ConnectionID = connectionID cmd.allocationChange.Size = cmd.fileChanger.Size cmd.allocationChange.Operation = constants.FileOperationInsert - - connectionObj.Size = allocationSize - + logging.Logger.Info("Chunk processed") return result, nil } // ProcessThumbnail flush thumbnail file to FileStorage if it has. -func (cmd *UploadFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error { - thumbfile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) +func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error { + connectionID := cmd.fileChanger.ConnectionID + if cmd.thumbHeader != nil { + defer cmd.thumbFile.Close() - if thumbHeader != nil { - defer thumbfile.Close() - - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: encryption.Hash(cmd.fileChanger.Path)} - thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, thumbInputData, thumbfile) + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) } cmd.fileChanger.ThumbnailSize = thumbOutputData.Size cmd.fileChanger.ThumbnailFilename = thumbInputData.Name + return allocation.SaveFileChanger(connectionID, &cmd.fileChanger.BaseFileChanger) } - - return nil + return common.ErrNoThumbnail } -func (cmd *UploadFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) { - for _, c := range connectionObj.Changes { - filePath, _ := c.GetOrParseAffectedFilePath() - if c.Operation != constants.FileOperationInsert || cmd.fileChanger.Path != filePath { - continue - } - - dbChangeProcessor := &allocation.UploadFileChanger{} - - err := dbChangeProcessor.Unmarshal(c.Input) - if err != nil { - logging.Logger.Error("reloadChange", zap.Error(err)) - } - - cmd.fileChanger.ThumbnailFilename = dbChangeProcessor.ThumbnailFilename - cmd.fileChanger.ThumbnailSize = dbChangeProcessor.ThumbnailSize - cmd.fileChanger.ThumbnailHash = dbChangeProcessor.ThumbnailHash - cmd.fileChanger.IsFinal = dbChangeProcessor.IsFinal - return +func (cmd *UploadFileCommand) reloadChange() { + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + if changer != nil { + cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename + cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize + cmd.fileChanger.ThumbnailHash = changer.ThumbnailHash } } @@ -246,3 +257,10 @@ func (cmd *UploadFileCommand) UpdateChange(ctx context.Context, connectionObj *a return connectionObj.Save(ctx) } + +func (cmd *UploadFileCommand) GetNumBlocks() int64 { + if cmd.fileChanger.IsFinal { + return int64(math.Ceil(float64(cmd.fileChanger.Size*1.0) / float64(cmd.fileChanger.ChunkSize))) + } + return 0 +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go index 7aae14228..3d9d7f38d 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_test.go @@ -233,6 +233,7 @@ func Test_GetAllocation(t *testing.T) { func(t *testing.T) { var mock = datastore.MockTheStore(t) test.mockSetup(mock) + allocation.Repo.DeleteAllocation(alloc.ID) if test.expectCommit { mock.ExpectCommit() } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index d55b55362..a16887b2a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -28,6 +28,8 @@ import ( "strings" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/go-openapi/runtime/middleware" "github.com/0chain/gosdk/constants" @@ -178,10 +180,10 @@ func setupHandlers(r *mux.Router) { Methods(http.MethodPost, http.MethodDelete, http.MethodOptions) r.HandleFunc("/v1/connection/commit/{allocation}", - RateLimitByCommmitRL(common.ToStatusCode(WithStatusConnection(CommitHandler)))) + RateLimitByCommmitRL(common.ToStatusCode(WithStatusConnectionForWM(CommitHandler)))) r.HandleFunc("/v1/connection/rollback/{allocation}", - RateLimitByCommmitRL(common.ToStatusCode(WithStatusConnection(RollbackHandler)))) + RateLimitByCommmitRL(common.ToStatusCode(WithStatusConnectionForWM(RollbackHandler)))) //object info related apis r.HandleFunc("/allocation", @@ -215,20 +217,19 @@ func setupHandlers(r *mux.Router) { // Allowing admin api for debugging purpose only. Later on commented out line should be // uncommented and line below it should be deleted - // r.HandleFunc("/_debug", common.AuthenticateAdmin(common.ToJSONResponse(DumpGoRoutines))) - r.HandleFunc("/_debug", RateLimitByCommmitRL(common.ToJSONResponse(DumpGoRoutines))) - // r.HandleFunc("/_config", common.AuthenticateAdmin(common.ToJSONResponse(GetConfig))) - r.HandleFunc("/_config", RateLimitByCommmitRL(common.ToJSONResponse(GetConfig))) + r.HandleFunc("/_debug", common.AuthenticateAdmin(common.ToJSONResponse(DumpGoRoutines))) + // r.HandleFunc("/_debug", RateLimitByCommmitRL(common.ToJSONResponse(DumpGoRoutines))) + r.HandleFunc("/_config", common.AuthenticateAdmin(common.ToJSONResponse(GetConfig))) + // r.HandleFunc("/_config", RateLimitByCommmitRL(common.ToJSONResponse(GetConfig))) // r.HandleFunc("/_stats", common.AuthenticateAdmin(StatsHandler)) r.HandleFunc("/_stats", RateLimitByCommmitRL(StatsHandler)) - // r.HandleFunc("/_statsJSON", common.AuthenticateAdmin(common.ToJSONResponse(stats.StatsJSONHandler))) - r.HandleFunc("/_statsJSON", RateLimitByCommmitRL(common.ToJSONResponse(stats.StatsJSONHandler))) + + r.HandleFunc("/_logs", RateLimitByCommmitRL(common.ToJSONResponse(GetLogs))) + // r.HandleFunc("/_cleanupdisk", common.AuthenticateAdmin(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) // r.HandleFunc("/_cleanupdisk", RateLimitByCommmitRL(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) - // r.HandleFunc("/getstats", common.AuthenticateAdmin(common.ToJSONResponse(stats.GetStatsHandler))) - r.HandleFunc("/getstats", RateLimitByCommmitRL(common.ToJSONResponse(WithReadOnlyConnection(stats.GetStatsHandler)))) - // r.HandleFunc("/challengetimings", common.AuthenticateAdmin(common.ToJSONResponse(GetChallengeTimings))) - r.HandleFunc("/challengetimings", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTimings))) + r.HandleFunc("/challengetimings", common.AuthenticateAdmin(common.ToJSONResponse(GetChallengeTimings))) + // r.HandleFunc("/challengetimings", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTimings))) r.HandleFunc("/challenge-timings-by-challengeId", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTiming))) //marketplace related @@ -740,7 +741,7 @@ func StatsHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, []byte(err.Error())) return } - + w.Header().Set("Content-Type", "application/json") writeResponse(w, statsJson) return @@ -783,6 +784,10 @@ func GetConfig(ctx context.Context, r *http.Request) (interface{}, error) { return config.Configuration, nil } +func GetLogs(ctx context.Context, r *http.Request) (interface{}, error) { + return transaction.Last50Transactions, nil +} + func CleanupDiskHandler(ctx context.Context, r *http.Request) (interface{}, error) { err := CleanupDiskFiles(ctx) diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go index 9a7d67e99..f716a7eae 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_common.go +++ b/code/go/0chain.net/blobbercore/handler/handler_common.go @@ -7,11 +7,14 @@ import ( "net/http" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/core/build" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/gosdk/zcncore" + "github.com/gorilla/mux" "go.uber.org/zap" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -111,14 +114,26 @@ func GetBlobberInfoJson() BlobberInfo { return blobberInfo } -func WithStatusConnection(handler common.StatusCodeResponderF) common.StatusCodeResponderF { +// Should only be used for handlers where the writemarker is submitted +func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.StatusCodeResponderF { return func(ctx context.Context, r *http.Request) (resp interface{}, statusCode int, err error) { ctx = GetMetaDataStore().CreateTransaction(ctx) + var vars = mux.Vars(r) + allocationID := vars["allocation_id"] + if allocationID != "" { + // Lock will compete with other CommitWrites and Challenge validation + var allocationObj *allocation.Allocation + mutex := lock.GetMutex(allocationObj.TableName(), allocationID) + Logger.Info("Locking allocation", zap.String("allocation_id", allocationID)) + mutex.Lock() + defer mutex.Unlock() + } + tx := GetMetaDataStore().GetTransaction(ctx) resp, statusCode, err = handler(ctx, r) defer func() { if err != nil { - var rollErr = GetMetaDataStore().GetTransaction(ctx). + var rollErr = tx. Rollback().Error if rollErr != nil { Logger.Error("couldn't rollback", zap.Error(err)) @@ -130,7 +145,7 @@ func WithStatusConnection(handler common.StatusCodeResponderF) common.StatusCode Logger.Error("Error in handling the request." + err.Error()) return } - err = GetMetaDataStore().GetTransaction(ctx).Commit().Error + err = tx.Commit().Error if err != nil { return resp, statusCode, common.NewErrorf("commit_error", "error committing to meta store: %v", err) diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go index 050958121..47dc95c3a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_download_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go @@ -4,6 +4,7 @@ package handler import ( + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -301,6 +302,7 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) @@ -478,10 +480,10 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) - r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Auth-Token", base64.StdEncoding.EncodeToString([]byte(authTicket))) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -559,10 +561,10 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) - r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Auth-Token", base64.StdEncoding.EncodeToString([]byte(authTicket))) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -673,10 +675,10 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) - r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Auth-Token", base64.StdEncoding.EncodeToString([]byte(authTicket))) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -793,10 +795,10 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) - r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Auth-Token", base64.StdEncoding.EncodeToString([]byte(authTicket))) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -912,10 +914,10 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Verify-Download", fmt.Sprint(true)) r.Header.Set("X-Connection-ID", connectionID) r.Header.Set("X-Mode", DownloadContentFull) - r.Header.Set("X-Auth-Token", authTicket) + r.Header.Set("X-Auth-Token", base64.StdEncoding.EncodeToString([]byte(authTicket))) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -991,7 +993,7 @@ func TestHandlers_Download(t *testing.T) { t.Run(test.name, func(t *testing.T) { mock := datastore.MockTheStore(t) test.setupDbMock(mock) - + allocation.Repo.DeleteAllocation(alloc.ID) if test.begin != nil { test.begin() } @@ -1005,9 +1007,7 @@ func TestHandlers_Download(t *testing.T) { m := make(map[string]interface{}) err = json.Unmarshal(data, &m) require.NoError(t, err) - if test.wantCode != http.StatusOK || test.wantBody != "" { - fmt.Println("fprint", test.args.w.Body.String()) var body string if m["Data"] != nil { body = m["Data"].(string) diff --git a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go index bdca11c56..79ea68676 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go @@ -251,7 +251,7 @@ func TestHandlers_ObjectTree(t *testing.T) { t.Run(test.name, func(t *testing.T) { mock := datastore.MockTheStore(t) test.setupDbMock(mock) - + allocation.Repo.DeleteAllocation(alloc.ID) if test.begin != nil { test.begin() } diff --git a/code/go/0chain.net/blobbercore/handler/handler_share_test.go b/code/go/0chain.net/blobbercore/handler/handler_share_test.go index 373c940bf..9a288997a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_share_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_share_test.go @@ -534,7 +534,7 @@ func TestHandlers_Share(t *testing.T) { t.Run(test.name, func(t *testing.T) { mock := datastore.MockTheStore(t) test.setupDbMock(mock) - + allocation.Repo.DeleteAllocation(alloc.ID) if test.begin != nil { test.begin() } diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 1c7343e20..d7a7e7632 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -28,6 +28,7 @@ import ( "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + blobConfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -72,6 +73,9 @@ func init() { resetMockFileBlock() chain.SetServerChain(&chain.Chain{}) config.Configuration.SignatureScheme = "bls0chain" + blobConfig.Configuration.BlockLimitDaily = 1562500 + blobConfig.Configuration.BlockLimitRequest = 500 + blobConfig.StorageSCConfig.MaxFileSize = 1024 * 1024 * 1024 * 1024 * 5 logging.Logger = zap.NewNop() ConfigRateLimits() @@ -354,7 +358,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q := url.Query() formFieldByt, err := json.Marshal( &allocation.UploadFileChanger{ - BaseFileChanger: allocation.BaseFileChanger{Path: path}}) + BaseFileChanger: allocation.BaseFileChanger{Path: path, ConnectionID: connectionID}}) if err != nil { t.Fatal(err) } @@ -640,7 +644,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ) lookUpHash := reference.GetReferenceLookup(alloc.ID, path) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT "id","name","path","hash","size","validation_root","fixed_merkle_root" FROM "reference_objects" WHERE`)). + mock.ExpectQuery(regexp.QuoteMeta(`SELECT "id","name","path","hash","size","validation_root","fixed_merkle_root","type" FROM "reference_objects" WHERE`)). WithArgs(lookUpHash). WillReturnRows( sqlmock.NewRows([]string{"type"}). @@ -769,7 +773,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { q := url.Query() formFieldByt, err := json.Marshal( &allocation.UploadFileChanger{ - BaseFileChanger: allocation.BaseFileChanger{Path: path, Size: size}}) + BaseFileChanger: allocation.BaseFileChanger{Path: path, Size: size, ConnectionID: connectionID}}) if err != nil { t.Fatal(err) } @@ -868,7 +872,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { t.Run(test.name, func(t *testing.T) { mock := datastore.MockTheStore(t) test.setupDbMock(mock) - + allocation.Repo.DeleteAllocation(alloc.ID) if test.begin != nil { test.begin() } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 99c27618e..fb0ec7d83 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -2,6 +2,7 @@ package handler import ( "context" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -13,6 +14,7 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/gosdk/constants" @@ -24,7 +26,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "go.uber.org/zap" @@ -277,6 +279,10 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewError("download_file", "invalid client") } + if ok := CheckBlacklist(clientID); ok { + return nil, common.NewError("blacklisted_client", "Client is blacklisted: "+clientID) + } + alloc, err := fsh.verifyAllocation(ctx, allocationID, allocationTx, false) if err != nil { return nil, common.NewErrorf("download_file", "invalid allocation id passed: %v", err) @@ -287,6 +293,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, err } + if dr.NumBlocks > config.Configuration.BlockLimitRequest { + return nil, common.NewErrorf("download_file", "too many blocks requested: %v, max limit is %v", dr.NumBlocks, config.Configuration.BlockLimitRequest) + } + + dailyBlocksConsumed := getDailyBlocks(clientID) + if dailyBlocksConsumed+dr.NumBlocks > config.Configuration.BlockLimitDaily { + return nil, common.NewErrorf("download_file", "daily block limit reached: %v, max limit is %v", dailyBlocksConsumed, config.Configuration.BlockLimitDaily) + } + fileref, err := reference.GetReferenceByLookupHash(ctx, alloc.ID, dr.PathHash) if err != nil { return nil, common.NewErrorf("download_file", "invalid file path: %v", err) @@ -302,12 +317,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i var shareInfo *reference.ShareInfo if !isOwner { - authTokenString := dr.AuthToken - if authTokenString == "" { + if dr.AuthToken == "" { return nil, common.NewError("invalid_authticket", "authticket is required") } + authTokenString, err := base64.StdEncoding.DecodeString(dr.AuthToken) + if err != nil { + return nil, common.NewError("invalid_authticket", err.Error()) + } - if authToken, err = fsh.verifyAuthTicket(ctx, authTokenString, alloc, fileref, clientID, false); authToken == nil { + if authToken, err = fsh.verifyAuthTicket(ctx, string(authTokenString), alloc, fileref, clientID, false); authToken == nil { return nil, common.NewErrorf("invalid_authticket", "cannot verify auth ticket: %v", err) } @@ -366,6 +384,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i IsPrecommit: fromPreCommit, } + logging.Logger.Info("calling GetFileBlock for thumb", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) @@ -385,6 +404,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i VerifyDownload: dr.VerifyDownload, IsPrecommit: fromPreCommit, } + logging.Logger.Info("calling GetFileBlock", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) if err != nil { return nil, common.NewErrorf("download_file", "couldn't get file block: %v", err) @@ -415,7 +435,14 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i } fileDownloadResponse.Data = chunkData - reference.FileBlockDownloaded(ctx, fileref.ID) + reference.FileBlockDownloaded(ctx, fileref, dr.NumBlocks) + go func() { + addDailyBlocks(clientID, dr.NumBlocks) + AddDownloadedData(clientID, dr.NumBlocks) + }() + if !dr.VerifyDownload { + return fileDownloadResponse.Data, nil + } return fileDownloadResponse, nil } @@ -476,6 +503,14 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b clientKey := ctx.Value(constants.ContextKeyClientKey).(string) clientKeyBytes, _ := hex.DecodeString(clientKey) + if clientID == "" || clientKey == "" { + return nil, common.NewError("invalid_parameters", "Please provide clientID and clientKey") + } + + if ok := CheckBlacklist(clientID); ok { + return nil, common.NewError("blacklisted_client", "Client is blacklisted: "+clientID) + } + allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -493,11 +528,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } - // Lock will compete with other CommitWrites and Challenge validation - mutex := lock.GetMutex(allocationObj.TableName(), allocationID) - mutex.Lock() - defer mutex.Unlock() - elapsedGetLock := time.Since(startTime) - elapsedAllocation err = checkPendingMarkers(ctx, allocationObj.ID) @@ -507,23 +537,22 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { // might be good to check if blobber already has stored writemarker return nil, common.NewErrorf("invalid_parameters", "Invalid connection id. Connection id was not found: %v", err) } if len(connectionObj.Changes) == 0 { + if connectionObj.Status == allocation.NewConnection { + return nil, common.NewError("invalid_parameters", + "Invalid connection id. Connection not found.") + } return nil, common.NewError("invalid_parameters", "Invalid connection id. Connection does not have any changes.") } elapsedGetConnObj := time.Since(startTime) - elapsedAllocation - elapsedGetLock - if clientID == "" || clientKey == "" { - return nil, common.NewError("invalid_parameters", "Please provide clientID and clientKey") - } - if allocationObj.OwnerID != clientID || encryption.Hash(clientKeyBytes) != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } @@ -603,20 +632,19 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - err = connectionObj.ApplyChanges( + rootRef, err := connectionObj.ApplyChanges( ctx, writeMarker.AllocationRoot, writeMarker.Timestamp, fileIDMeta) if err != nil { Logger.Error("Error applying changes", zap.Error(err)) return nil, err } + if !rootRef.IsPrecommit { + return nil, common.NewError("no_root_change", "No change in root ref") + } elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - rootRef, err := reference.GetLimitedRefFieldsByPath(ctx, allocationID, "/", []string{"hash", "file_meta_hash"}) - if err != nil { - return nil, err - } allocationRoot := rootRef.Hash fileMetaRoot := rootRef.FileMetaHash if allocationRoot != writeMarker.AllocationRoot { @@ -645,7 +673,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b writemarkerEntity.ClientPublicKey = clientKey db := datastore.GetStore().GetTransaction(ctx) - + writemarkerEntity.Latest = true if err = db.Create(writemarkerEntity).Error; err != nil { return nil, common.NewError("write_marker_error", "Error persisting the write marker") } @@ -655,7 +683,22 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size - if err = allocation.Repo.Save(ctx, allocationObj); err != nil { + updateMap := map[string]interface{}{ + "allocation_root": allocationRoot, + "file_meta_root": fileMetaRoot, + "used_size": allocationObj.UsedSize, + "blobber_size_used": allocationObj.BlobberSizeUsed, + "is_redeem_required": true, + } + updateOption := func(a *allocation.Allocation) { + a.AllocationRoot = allocationRoot + a.FileMetaRoot = fileMetaRoot + a.IsRedeemRequired = true + a.BlobberSizeUsed = allocationObj.BlobberSizeUsed + a.UsedSize = allocationObj.UsedSize + } + + if err = allocation.Repo.UpdateAllocation(ctx, allocationObj, updateMap, updateOption); err != nil { return nil, common.NewError("allocation_write_error", "Error persisting the allocation object") } @@ -674,8 +717,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("write_marker_error", "Error redeeming the write marker") } - result.Changes = connectionObj.Changes - connectionObj.DeleteChanges(ctx) db.Model(connectionObj).Updates(allocation.AllocationChangeCollector{Status: allocation.CommittedConnection}) @@ -690,6 +731,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b db.Delete(connectionObj) go allocation.DeleteConnectionObjEntry(connectionID) + go AddWriteMarkerCount(clientID, 1) Logger.Info("[commit]"+commitOperation, zap.String("alloc_id", allocationID), @@ -759,7 +801,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, common.NewError("meta_error", "Error reading metadata for connection") } - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -774,7 +816,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i allocationChange.Size = 0 allocationChange.Operation = constants.FileOperationRename dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, Path: objectRef.Path} + AllocationID: connectionObj.AllocationID, Path: objectRef.Path, Type: objectRef.Type} dfc.NewName = new_name connectionObj.AddChange(allocationChange, dfc) @@ -784,7 +826,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, common.NewError("connection_write_error", "Error writing the connection meta data") } - result := &blobberhttp.UploadResult{} + result := &allocation.UploadResult{} result.Filename = new_name result.Hash = objectRef.Hash result.ValidationRoot = objectRef.ValidationRoot @@ -894,7 +936,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("connection_write_error", "Error writing the connection meta data") } - result := &blobberhttp.UploadResult{} + result := &allocation.UploadResult{} result.Filename = objectRef.Name result.Hash = objectRef.Hash result.ValidationRoot = objectRef.ValidationRoot @@ -1009,7 +1051,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("connection_write_error", "Error writing the connection meta data") } - result := &blobberhttp.UploadResult{} + result := &allocation.UploadResult{} result.Filename = objectRef.Name result.Hash = objectRef.Hash result.ValidationRoot = objectRef.ValidationRoot @@ -1018,7 +1060,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int return result, nil } -func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, connectionObj *allocation.AllocationChangeCollector) (*blobberhttp.UploadResult, error) { +func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, connectionObj *allocation.AllocationChangeCollector) (*allocation.UploadResult, error) { path := r.FormValue("path") if path == "" { @@ -1046,7 +1088,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn connectionObj.AddChange(allocationChange, dfc) - result := &blobberhttp.UploadResult{} + result := &allocation.UploadResult{} result.Filename = fileRef.Name result.Hash = fileRef.Hash result.ValidationRoot = fileRef.ValidationRoot @@ -1059,7 +1101,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn return nil, common.NewError("invalid_file", "File does not exist at path") } -func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blobberhttp.UploadResult, error) { +func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*allocation.UploadResult, error) { allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) clientID := ctx.Value(constants.ContextKeyClient).(string) @@ -1089,7 +1131,7 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo Logger.Error("Error file reference", zap.Error(err)) } - result := &blobberhttp.UploadResult{ + result := &allocation.UploadResult{ Filename: dirPath, } @@ -1147,10 +1189,8 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*blo } // WriteFile stores the file into the blobber files system from the HTTP request -func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blobberhttp.UploadResult, error) { - +func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*allocation.UploadResult, error) { startTime := time.Now() - if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST / PUT / DELETE / PATCH instead") } @@ -1158,13 +1198,33 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) clientID := ctx.Value(constants.ContextKeyClient).(string) + connectionID, ok := common.GetField(r, "connection_id") + if !ok { + return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + } + if clientID == "" { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + if ok := CheckBlacklist(clientID); ok { + return nil, common.NewError("blacklisted_client", "Client is blacklisted: "+clientID) + } + elapsedParseForm := time.Since(startTime) + st := time.Now() + connectionProcessor := allocation.GetConnectionProcessor(connectionID) + if connectionProcessor == nil { + connectionProcessor = allocation.CreateConnectionProcessor(connectionID) + } + + elapsedGetConnectionProcessor := time.Since(st) + st = time.Now() allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } + connectionProcessor.ClientID = clientID - elapsedAllocation := time.Since(startTime) + elapsedAllocation := time.Since(st) if r.Method == http.MethodPost && !allocationObj.CanUpload() { return nil, common.NewError("prohibited_allocation_file_options", "Cannot upload data to this allocation.") @@ -1178,18 +1238,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("prohibited_allocation_file_options", "Cannot delete data in this allocation.") } - st := time.Now() - allocationID := allocationObj.ID - cmd := createFileCommand(r) - err = cmd.IsValidated(ctx, r, allocationObj, clientID) - - if err != nil { - return nil, err - } - - elapsedValidate := time.Since(st) st = time.Now() - publicKey := allocationObj.OwnerPublicKey valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), publicKey) @@ -1198,66 +1247,32 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("invalid_signature", "Invalid signature") } - connectionID, ok := common.GetField(r, "connection_id") - if !ok { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - elapsedRef := time.Since(st) - st = time.Now() - - connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { - return nil, common.NewError("meta_error", "Error reading metadata for connection") - } - - elapsedAllocationChanges := time.Since(st) - - Logger.Info("[upload] Processing content for allocation and connection", - zap.String("allocationID", allocationID), - zap.String("connectionID", connectionID), - ) - st = time.Now() - result, err := cmd.ProcessContent(ctx, r, allocationObj, connectionObj) + elapsedVerifySig := time.Since(st) + allocationID := allocationObj.ID + cmd := createFileCommand(r) + err = cmd.IsValidated(ctx, r, allocationObj, clientID) if err != nil { return nil, err } - Logger.Info("[upload] Content processed for allocation and connection", - zap.String("allocationID", allocationID), - zap.String("connectionID", connectionID), - ) - - err = cmd.ProcessThumbnail(ctx, r, allocationObj, connectionObj) - - if err != nil { - return nil, err + result := allocation.UploadResult{ + Filename: cmd.GetPath(), } - - elapsedProcess := time.Since(st) - st = time.Now() - err = cmd.UpdateChange(ctx, connectionObj) - - if err != nil { - Logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", err.Error()) //"Error writing the connection meta data") + blocks := cmd.GetNumBlocks() + if blocks > 0 { + go AddUploadedData(clientID, blocks) } - - elapsedUpdateChange := time.Since(st) - Logger.Info("[upload]elapsed", zap.String("alloc_id", allocationID), zap.String("file", cmd.GetPath()), + zap.Duration("parse_form", elapsedParseForm), + zap.Duration("get_processor", elapsedGetConnectionProcessor), zap.Duration("get_alloc", elapsedAllocation), - zap.Duration("validate", elapsedValidate), - zap.Duration("ref", elapsedRef), - zap.Duration("load_changes", elapsedAllocationChanges), - zap.Duration("process", elapsedProcess), - zap.Duration("update_changes", elapsedUpdateChange), - zap.Duration("total", time.Since(startTime)), - ) - + zap.Duration("sig", elapsedVerifySig), + zap.Duration("validate", time.Since(st)), + zap.Duration("total", time.Since(startTime))) return &result, nil + } func sanitizeString(input string) string { @@ -1280,8 +1295,12 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob clientID := ctx.Value(constants.ContextKeyClient).(string) clientKey := ctx.Value(constants.ContextKeyClientKey).(string) clientKeyBytes, _ := hex.DecodeString(clientKey) + var ( + allocationObj *allocation.Allocation + err error + ) - allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + allocationObj, err = fsh.verifyAllocation(ctx, allocationId, allocationTx, false) if err != nil { Logger.Error("Error in verifying allocation", zap.Error(err)) return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) @@ -1299,10 +1318,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob if !ok { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } - // Lock will compete with other CommitWrites and Challenge validation - mutex := lock.GetMutex(allocationObj.TableName(), allocationID) - mutex.Lock() - defer mutex.Unlock() elapsedGetLock := time.Since(startTime) - elapsedAllocation @@ -1416,17 +1431,29 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob alloc.UsedSize -= latestWriteMarkerEntity.WM.Size alloc.AllocationRoot = allocationRoot alloc.FileMetaRoot = fileMetaRoot - sendWM := !alloc.IsRedeemRequired - if alloc.IsRedeemRequired { - writemarkerEntity.Status = writemarker.Rollbacked - alloc.IsRedeemRequired = false - } + alloc.IsRedeemRequired = true + updateMap := map[string]interface{}{ + "blobber_size_used": alloc.BlobberSizeUsed, + "used_size": alloc.UsedSize, + "allocation_root": alloc.AllocationRoot, + "file_meta_root": alloc.FileMetaRoot, + "is_redeem_required": true, + } + + updateOption := func(a *allocation.Allocation) { + a.BlobberSizeUsed = alloc.BlobberSizeUsed + a.UsedSize = alloc.UsedSize + a.AllocationRoot = alloc.AllocationRoot + a.FileMetaRoot = alloc.FileMetaRoot + a.IsRedeemRequired = alloc.IsRedeemRequired + } + writemarkerEntity.Latest = true err = txn.Create(writemarkerEntity).Error if err != nil { txn.Rollback() return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) } - if err = allocation.Repo.Save(c, alloc); err != nil { + if err = allocation.Repo.UpdateAllocation(c, alloc, updateMap, updateOption); err != nil { txn.Rollback() return &result, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) } @@ -1435,11 +1462,9 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob if err != nil { return &result, common.NewError("allocation_commit_error", "Error committing the transaction "+err.Error()) } - if sendWM { - err = writemarkerEntity.SendToChan(ctx) - if err != nil { - return nil, common.NewError("write_marker_error", "Error redeeming the write marker") - } + err = writemarkerEntity.SendToChan(ctx) + if err != nil { + return nil, common.NewError("write_marker_error", "Error redeeming the write marker") } err = allocation.CommitRollback(allocationID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 988e1c6ab..3edfafbf8 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -3,6 +3,7 @@ package handler import ( "context" "database/sql/driver" + "encoding/base64" "encoding/json" "fmt" "log" @@ -10,7 +11,6 @@ import ( "net/http/httptest" "time" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -128,15 +128,16 @@ func TestDownloadFile(t *testing.T) { require.NoError(t, authTicket.Sign()) require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) authTicketBytes, _ := json.Marshal(authTicket) - req.Header.Set("X-Auth-Token", string(authTicketBytes)) + auth := base64.StdEncoding.EncodeToString(authTicketBytes) + req.Header.Set("X-Auth-Token", auth) } if len(p.inData.contentMode) > 0 { req.Header.Set("X-Mode", p.inData.contentMode) } } - makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { - return func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { + makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string) ([]byte, error) { + return func(scAddress string, relativePath string, params map[string]string) ([]byte, error) { require.New(t) require.EqualValues(t, scAddress, transaction.STORAGE_CONTRACT_ADDRESS) switch relativePath { @@ -460,6 +461,7 @@ func TestDownloadFile(t *testing.T) { datastore.MocketTheStore(t, mocketLogging) setupInMock(t, test.parameters) setupOutMock(t, test.parameters) + allocation.Repo.DeleteAllocation(mockAllocationId) ctx := setupCtx(test.parameters) ctx = context.WithValue(ctx, constants.ContextKeyAllocationID, mockAllocationId) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 8e3cfad68..ec10bf827 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -9,8 +9,6 @@ import ( "regexp" "strconv" - "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" @@ -50,8 +48,6 @@ func (fsh *StorageHandler) verifyAllocation(ctx context.Context, allocationID, a "verifying allocation transaction error: %v", err) } - logging.Logger.Info("verifyAllocation", zap.Any("alloc", alloc), zap.Any("now", common.Now())) - if alloc.Expiration < common.Now() { return nil, common.NewError("verify_allocation", "use of expired allocation") @@ -252,7 +248,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i } result := fileref.GetListingData(ctx) - fileStats, err := reference.GetFileStats(ctx, fileref.ID) + fileStats, err := reference.GetFileStats(ctx, fileref) if err != nil { return nil, common.NewError("bad_db_operation", "Error retrieving file stats. "+err.Error()) } @@ -293,11 +289,11 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* if err != nil { return nil, err } - + _, ok := common.GetField(r, "list") escapedPathHash := sanitizeString(pathHash) Logger.Info("Path Hash for list dir :" + escapedPathHash) - fileref, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "path", "lookup_hash", "type", "name"}) + fileref, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "path", "lookup_hash", "type", "name", "file_meta_hash", "parent_path"}) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { // `/` always is valid even it doesn't exists in db. so ignore RecordNotFound error @@ -309,9 +305,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } } - authTokenString := r.FormValue("auth_token") + authTokenString, _ := common.GetField(r, "auth_token") if clientID != allocationObj.OwnerID || len(authTokenString) > 0 { - authToken, err := fsh.verifyAuthTicket(ctx, r.FormValue("auth_token"), allocationObj, fileref, clientID, true) + authToken, err := fsh.verifyAuthTicket(ctx, authTokenString, allocationObj, fileref, clientID, true) if err != nil { return nil, err } @@ -320,6 +316,27 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } } + if !ok { + var listResult blobberhttp.ListResult + listResult.AllocationRoot = allocationObj.AllocationRoot + if fileref == nil { + fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID} + } + if fileref.Type == reference.FILE { + parent, err := reference.GetReference(ctx, allocationID, fileref.ParentPath) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid path. Parent dir of file not found. "+err.Error()) + } + fileref = parent + } + listResult.Meta = fileref.GetListingData(ctx) + if clientID != allocationObj.OwnerID { + delete(listResult.Meta, "path") + } + listResult.Entities = make([]map[string]interface{}, 0) + return &listResult, nil + } + // when '/' is not available in database we ignore 'record not found' error. which results into nil fileRef // to handle that condition use filePath '/' while file ref is nil and path is '/' filePath := path @@ -404,9 +421,11 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req } else { latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) + Logger.Error("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) + return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation. "+err.Error()) } if latestWM == nil { + Logger.Info("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) return nil, common.NewError("latest_write_marker_read_error", "Latest write marker not found for allocation.") } if latestWM.WM.PreviousAllocationRoot != "" { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 2634b148e..1ba88c539 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -17,6 +17,8 @@ import ( func SetupWorkers(ctx context.Context) { go startCleanupTempFiles(ctx) + go startDownloadLimitCleanup(ctx) + go startBlackListWorker(ctx) } func CleanupDiskFiles(ctx context.Context) error { diff --git a/code/go/0chain.net/blobbercore/readmarker/protocol.go b/code/go/0chain.net/blobbercore/readmarker/protocol.go index 2603cad2b..e568f0070 100644 --- a/code/go/0chain.net/blobbercore/readmarker/protocol.go +++ b/code/go/0chain.net/blobbercore/readmarker/protocol.go @@ -5,7 +5,6 @@ import ( "encoding/json" "time" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" @@ -82,8 +81,7 @@ func GetLatestReadMarkerEntityFromChain(clientID, allocID string) (*ReadMarker, } latestRMBytes, err := transaction.MakeSCRestAPICall( - transaction.STORAGE_CONTRACT_ADDRESS, "/latestreadmarker", params, - chain.GetServerChain()) + transaction.STORAGE_CONTRACT_ADDRESS, "/latestreadmarker", params) if err != nil { return nil, err diff --git a/code/go/0chain.net/blobbercore/readmarker/worker.go b/code/go/0chain.net/blobbercore/readmarker/worker.go index f123e0179..c0515680e 100644 --- a/code/go/0chain.net/blobbercore/readmarker/worker.go +++ b/code/go/0chain.net/blobbercore/readmarker/worker.go @@ -9,7 +9,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" @@ -32,8 +31,7 @@ func redeemReadMarker(ctx context.Context, rmEntity *ReadMarkerEntity) (err erro latestRM := ReadMarker{BlobberID: rmEntity.LatestRM.BlobberID, ClientID: rmEntity.LatestRM.ClientID} latestRMBytes, err := transaction.MakeSCRestAPICall( - transaction.STORAGE_CONTRACT_ADDRESS, "/latestreadmarker", params, - chain.GetServerChain()) + transaction.STORAGE_CONTRACT_ADDRESS, "/latestreadmarker", params) if err != nil { logging.Logger.Error("Error from sc rest api call", zap.Error(err)) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go new file mode 100644 index 000000000..37c3cc808 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -0,0 +1,49 @@ +package reference + +import ( + "context" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" +) + +type QueryCollector interface { + CreateRefRecord(ref *Ref) + DeleteRefRecord(ref *Ref) + Finalize(ctx context.Context) error +} + +type dbCollector struct { + createdRefs []*Ref + deletedRefs []*Ref +} + +func NewCollector(changes int) QueryCollector { + return &dbCollector{ + createdRefs: make([]*Ref, 0, changes*4), + deletedRefs: make([]*Ref, 0, changes*4), + } +} + +func (dc *dbCollector) CreateRefRecord(ref *Ref) { + dc.createdRefs = append(dc.createdRefs, ref) +} + +func (dc *dbCollector) DeleteRefRecord(ref *Ref) { + dc.deletedRefs = append(dc.deletedRefs, ref) +} + +func (dc *dbCollector) Finalize(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + if len(dc.deletedRefs) > 0 { + err := db.Delete(dc.deletedRefs).Error + if err != nil { + return err + } + } + err := db.Create(dc.createdRefs).Error + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/reference/filestats.go b/code/go/0chain.net/blobbercore/reference/filestats.go index 3477b64d0..539c974df 100644 --- a/code/go/0chain.net/blobbercore/reference/filestats.go +++ b/code/go/0chain.net/blobbercore/reference/filestats.go @@ -56,50 +56,43 @@ func NewFileCreated(ctx context.Context, refID int64) { stats := &FileStats{RefID: refID} stats.NumBlockDownloads = 0 stats.NumUpdates = 1 - db.Save(&stats) + db.Save(stats) } func FileUpdated(ctx context.Context, refID, newRefID int64) { if refID == 0 { return } - db := datastore.GetStore().GetTransaction(ctx) - stats, err := GetFileStats(ctx, refID) - if err != nil { - logging.Logger.Error("FileUpdatedGetFileStats", zap.Error(err)) - return - } else { - logging.Logger.Info("FileUpdatedGetFileStats", zap.Any("stats", stats)) - } - db.Delete(&FileStats{}, "id=?", stats.ID) - newStats := &FileStats{RefID: newRefID} - newStats.NumUpdates = stats.NumUpdates + 1 - newStats.NumBlockDownloads = stats.NumBlockDownloads - newStats.SuccessChallenges = stats.SuccessChallenges - newStats.FailedChallenges = stats.FailedChallenges - newStats.LastChallengeResponseTxn = stats.LastChallengeResponseTxn - newStats.WriteMarkerRedeemTxn = stats.WriteMarkerRedeemTxn - newStats.OnChain = stats.OnChain - err = db.Create(newStats).Error - if err != nil { - logging.Logger.Error("FileUpdatedCreate", zap.Error(err)) - } + // db := datastore.GetStore().GetTransaction(ctx) + // stats, err := GetFileStats(ctx, refID) + // if err != nil { + // logging.Logger.Error("FileUpdatedGetFileStats", zap.Error(err)) + // return + // } else { + // logging.Logger.Info("FileUpdatedGetFileStats", zap.Any("stats", stats)) + // } + // db.Delete(&FileStats{}, "id=?", stats.ID) + // newStats := &FileStats{RefID: newRefID} + // newStats.NumUpdates = stats.NumUpdates + 1 + // newStats.NumBlockDownloads = stats.NumBlockDownloads + // newStats.SuccessChallenges = stats.SuccessChallenges + // newStats.FailedChallenges = stats.FailedChallenges + // newStats.LastChallengeResponseTxn = stats.LastChallengeResponseTxn + // newStats.WriteMarkerRedeemTxn = stats.WriteMarkerRedeemTxn + // newStats.OnChain = stats.OnChain + // err = db.Create(newStats).Error + // if err != nil { + // logging.Logger.Error("FileUpdatedCreate", zap.Error(err)) + // } } -func FileBlockDownloaded(ctx context.Context, refID int64) { +func FileBlockDownloaded(ctx context.Context, ref *Ref, blocks int64) { db := datastore.GetStore().GetTransaction(ctx) - stats := &FileStats{RefID: refID} - db.Model(stats).Where(FileStats{RefID: refID}).Update("num_of_block_downloads", gorm.Expr("num_of_block_downloads + ?", 1)) + db.Model(ref).Update("num_of_block_downloads", gorm.Expr("num_of_block_downloads + ?", blocks)) } -func GetFileStats(ctx context.Context, refID int64) (*FileStats, error) { - db := datastore.GetStore().GetTransaction(ctx) - stats := &FileStats{RefID: refID} - err := db.Unscoped().Model(stats).Where(FileStats{RefID: refID}).Preload("Ref").First(stats).Error - if err != nil { - return nil, err - } - +func GetFileStats(ctx context.Context, ref *Ref) (*FileStats, error) { + stats := &FileStats{RefID: ref.ID, Ref: *ref, NumUpdates: ref.NumUpdates, NumBlockDownloads: ref.NumBlockDownloads} return stats, nil } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 5b8d42d5d..3e686b57d 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -34,8 +34,8 @@ type Ref struct { Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"` AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_parent_path_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"` - Name string `gorm:"column:name;size:100;not null;index:idx_name_gin:gin" dirlist:"name" filelist:"name"` - Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx" dirlist:"path" filelist:"path"` + Name string `gorm:"column:name;size:100;not null;index:idx_name_gin" dirlist:"name" filelist:"name"` // uses GIN tsvector index for full-text search + Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx;index:idx_path_gin_trgm" dirlist:"path" filelist:"path"` FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"` NumBlocks int64 `gorm:"column:num_of_blocks;not null;default:0" dirlist:"num_of_blocks" filelist:"num_of_blocks"` @@ -51,7 +51,7 @@ type Ref struct { ActualFileSize int64 `gorm:"column:actual_file_size;not null;default:0" dirlist:"actual_file_size" filelist:"actual_file_size"` ActualFileHashSignature string `gorm:"column:actual_file_hash_signature;size:64" filelist:"actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"` ActualFileHash string `gorm:"column:actual_file_hash;size:64;not null" filelist:"actual_file_hash"` - MimeType string `gorm:"column:mimetype;size:64;not null" filelist:"mimetype"` + MimeType string `gorm:"column:mimetype;size:255;not null" filelist:"mimetype"` AllocationRoot string `gorm:"column:allocation_root;size:64;not null"` ThumbnailSize int64 `gorm:"column:thumbnail_size;not null;default:0" filelist:"thumbnail_size"` ThumbnailHash string `gorm:"column:thumbnail_hash;size:64;not null" filelist:"thumbnail_hash"` @@ -65,10 +65,13 @@ type Ref struct { CreatedAt common.Timestamp `gorm:"column:created_at;index:idx_created_at,sort:desc" dirlist:"created_at" filelist:"created_at"` UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at,sort:desc;" dirlist:"updated_at" filelist:"updated_at"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion - IsPrecommit bool `gorm:"column:is_precommit;not null;default:false" filelist:"is_precommit" dirlist:"is_precommit"` - ChunkSize int64 `gorm:"column:chunk_size;not null;default:65536" dirlist:"chunk_size" filelist:"chunk_size"` - HashToBeComputed bool `gorm:"-"` + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion + IsPrecommit bool `gorm:"column:is_precommit;not null;default:false" filelist:"is_precommit" dirlist:"is_precommit"` + ChunkSize int64 `gorm:"column:chunk_size;not null;default:65536" dirlist:"chunk_size" filelist:"chunk_size"` + NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` + NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"` + HashToBeComputed bool `gorm:"-"` + prevID int64 `gorm:"-"` } // BeforeCreate Hook that gets executed to update create and update date @@ -404,7 +407,7 @@ func (r *Ref) GetHashData() string { return fmt.Sprintf("%s:%s:%s", r.AllocationID, r.Path, r.FileID) } -func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, error) { +func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { fr.FileMetaHash = encryption.Hash(fr.GetFileMetaHashData()) fr.Hash = encryption.Hash(fr.GetFileHashData()) fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) @@ -414,12 +417,12 @@ func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool) (string, er var err error if saveToDB && fr.HashToBeComputed { - err = fr.SaveFileRef(ctx) + err = fr.SaveFileRef(ctx, collector) } return fr.Hash, err } -func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (h string, err error) { +func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector QueryCollector) (h string, err error) { if !r.HashToBeComputed { h = r.Hash return @@ -429,7 +432,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (h string, er defer func() { if err == nil && saveToDB { - err = r.SaveDirRef(ctx) + err = r.SaveDirRef(ctx, collector) } }() @@ -441,7 +444,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (h string, er for i, childRef := range r.Children { if childRef.HashToBeComputed { - _, err := childRef.CalculateHash(ctx, saveToDB) + _, err := childRef.CalculateHash(ctx, saveToDB, collector) if err != nil { return "", err } @@ -466,11 +469,11 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (h string, er return r.Hash, err } -func (r *Ref) CalculateHash(ctx context.Context, saveToDB bool) (string, error) { +func (r *Ref) CalculateHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { if r.Type == DIRECTORY { - return r.CalculateDirHash(ctx, saveToDB) + return r.CalculateDirHash(ctx, saveToDB, collector) } - return r.CalculateFileHash(ctx, saveToDB) + return r.CalculateFileHash(ctx, saveToDB, collector) } func (r *Ref) AddChild(child *Ref) { @@ -527,65 +530,30 @@ func DeleteReference(ctx context.Context, refID int64, pathHash string) error { return db.Where("path_hash = ?", pathHash).Delete(&Ref{ID: refID}).Error } -func (r *Ref) SaveFileRef(ctx context.Context) error { - db := datastore.GetStore().GetTransaction(ctx) - toUpdateFileStat := r.IsPrecommit - prevID := r.ID +func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error { + r.prevID = r.ID + r.IsPrecommit = true + r.NumUpdates += 1 if r.ID > 0 { - err := db.Delete(&Ref{}, "id=?", r.ID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } - + deleteRef := &Ref{ID: r.ID} + collector.DeleteRefRecord(deleteRef) r.ID = 0 - r.IsPrecommit = true - err = db.Create(r).Error - if err != nil { - return err - } - - if toUpdateFileStat { - FileUpdated(ctx, prevID, r.ID) - } - } else { - r.IsPrecommit = true - err := db.Create(r).Error - if err != nil { - return err - } - NewFileCreated(ctx, r.ID) } + collector.CreateRefRecord(r) return nil } -func (r *Ref) SaveDirRef(ctx context.Context) error { - db := datastore.GetStore().GetTransaction(ctx) - toUpdateFileStat := r.IsPrecommit - prevID := r.ID +func (r *Ref) SaveDirRef(ctx context.Context, collector QueryCollector) error { + r.prevID = r.ID + r.IsPrecommit = true + r.NumUpdates += 1 if r.ID > 0 { - err := db.Delete(&Ref{}, "id=?", r.ID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } + deleteRef := &Ref{ID: r.ID} + collector.DeleteRefRecord(deleteRef) r.ID = 0 - r.IsPrecommit = true - err = db.Create(r).Error - if err != nil { - return err - } - - if toUpdateFileStat { - FileUpdated(ctx, prevID, r.ID) - } - } else { - r.IsPrecommit = true - err := db.Create(r).Error - if err != nil { - return err - } - NewDirCreated(ctx, r.ID) } + collector.CreateRefRecord(r) return nil } diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 633937f0c..3c158bfae 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -83,17 +83,44 @@ type BlobberStats struct { WriteMarkers WriteMarkersStat `json:"write_markers"` } +var fs *BlobberStats + +const statsHandlerPeriod = 30 * time.Minute + type AllocationId struct { Id string `json:"id"` } +func SetupStatsWorker(ctx context.Context) { + fs = &BlobberStats{} + go func() { + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + fs.loadBasicStats(ctx) + fs.loadDetailedStats(ctx) + fs.loadFailedChallengeList(ctx) + return common.NewError("rollback", "read_only") + }) + for { + select { + case <-ctx.Done(): + return + case <-time.After(statsHandlerPeriod): + newFs := &BlobberStats{} + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + newFs.loadBasicStats(ctx) + newFs.loadDetailedStats(ctx) + newFs.loadFailedChallengeList(ctx) + fs = newFs + return common.NewError("rollback", "read_only") + }) + } + } + }() +} + func LoadBlobberStats(ctx context.Context) *BlobberStats { - fs := &BlobberStats{} - fs.loadBasicStats(ctx) - fs.loadDetailedStats(ctx) fs.loadInfraStats(ctx) fs.loadDBStats() - fs.loadFailedChallengeList(ctx) return fs } @@ -211,14 +238,10 @@ func (bs *BlobberStats) loadStats(ctx context.Context) { const sel = ` COALESCE (SUM (reference_objects.size), 0) AS files_size, COALESCE (SUM (reference_objects.thumbnail_size), 0) AS thumbnails_size, - COALESCE (SUM (file_stats.num_of_block_downloads), 0) AS num_of_reads, + COALESCE (SUM (reference_objects.num_of_block_downloads), 0) AS num_of_reads, COALESCE (SUM (reference_objects.num_of_blocks), 0) AS num_of_block_writes, COUNT (*) AS num_of_writes` - const join = ` - INNER JOIN file_stats ON reference_objects.id = file_stats.ref_id - WHERE reference_objects.type = 'f'` - var ( db = datastore.GetStore().GetTransaction(ctx) row *sql.Row @@ -227,9 +250,14 @@ func (bs *BlobberStats) loadStats(ctx context.Context) { row = db.Table("reference_objects"). Select(sel). - Joins(join). + Where("reference_objects.type = 'f' AND reference_objects.deleted_at is NULL"). Row() + if row == nil { + Logger.Info("No rows found for blobber stats") + return + } + err = row.Scan(&bs.FilesSize, &bs.ThumbnailsSize, &bs.NumReads, &bs.BlockWrites, &bs.NumWrites) if err != nil && err != sql.ErrNoRows { @@ -269,17 +297,15 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { reference_objects.allocation_id, SUM(reference_objects.size) as files_size, SUM(reference_objects.thumbnail_size) as thumbnails_size, - SUM(file_stats.num_of_block_downloads) as num_of_reads, + SUM(reference_objects.num_of_block_downloads) as num_of_reads, SUM(reference_objects.num_of_blocks) as num_of_block_writes, COUNT(*) as num_of_writes, allocations.blobber_size AS allocated_size, allocations.expiration_date AS expiration_date`). - Joins(`INNER JOIN file_stats - ON reference_objects.id = file_stats.ref_id`). Joins(` INNER JOIN allocations ON allocations.id = reference_objects.allocation_id`). - Where(`reference_objects.type = 'f' AND allocations.finalized = 'f'`). + Where(`reference_objects.type = 'f' AND allocations.finalized = 'f' AND reference_objects.deleted_at is NULL`). Group(`reference_objects.allocation_id, allocations.expiration_date`). Group(`reference_objects.allocation_id, allocations.blobber_size`). Rows() @@ -306,21 +332,14 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { bs.AllocationStats = append(bs.AllocationStats, as) } - if err = rows.Err(); err != nil && err != sql.ErrNoRows { + if err = rows.Err(); err != nil && !errors.Is(err, sql.ErrNoRows) { Logger.Error("Error in scanning record for blobber stats", zap.Error(err)) return } - var count int64 - err = db.Table("reference_objects").Count(&count).Error - if err != nil { - Logger.Error("loadAllocationStats err", zap.Any("err", err)) - return - } - if requestData != nil { - pagination := GeneratePagination(requestData.Page, requestData.Limit, requestData.Offset, int(count)) + pagination := GeneratePagination(requestData.Page, requestData.Limit, requestData.Offset, len(bs.AllocationStats)) bs.AllocationListPagination = pagination } } @@ -447,47 +466,6 @@ func (bs *BlobberStats) loadAllocationChallengeStats(ctx context.Context) { } } -func loadAllocationList(ctx context.Context) (interface{}, error) { - var ( - allocations = make([]AllocationId, 0) - db = datastore.GetStore().GetTransaction(ctx) - rows *sql.Rows - err error - ) - - rows, err = db.Table("reference_objects"). - Select("reference_objects.allocation_id"). - Group("reference_objects.allocation_id"). - Rows() - - if err != nil { - Logger.Error("Error in getting the allocation list", zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to get allocation list from DB") - } - defer rows.Close() - - for rows.Next() { - var allocationId AllocationId - if err = rows.Scan(&allocationId.Id); err != nil { - Logger.Error("Error in scanning record for blobber allocations", - zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to scan allocation from DB") - } - allocations = append(allocations, allocationId) - } - - if err = rows.Err(); err != nil && err != sql.ErrNoRows { - Logger.Error("Error in scanning record for blobber allocations", - zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to scan allocations from DB") - } - - return allocations, nil -} - type ReadMarkerEntity struct { ReadCounter int64 `gorm:"column:counter" json:"counter"` LatestRedeemedRC int64 `gorm:"column:latest_redeemed_rc"` diff --git a/code/go/0chain.net/blobbercore/stats/handler.go b/code/go/0chain.net/blobbercore/stats/handler.go index 3e1349930..e6d7e7eea 100644 --- a/code/go/0chain.net/blobbercore/stats/handler.go +++ b/code/go/0chain.net/blobbercore/stats/handler.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" "go.uber.org/zap" ) @@ -241,6 +240,8 @@ const tpl = ` +

Last 50 Transactions

+

Allocation Stats

@@ -483,26 +484,3 @@ func StatsJSONHandler(ctx context.Context, r *http.Request) (interface{}, error) bs := LoadBlobberStats(ctx) return bs, nil } - -func GetStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) { - q := r.URL.Query() - ctx = context.WithValue(ctx, constants.ContextKeyAllocation, q.Get("allocation_id")) - allocationID := ctx.Value(constants.ContextKeyAllocation).(string) - bs := &BlobberStats{} - if allocationID != "" { - // TODO: Get only the allocation info from DB - bs.loadDetailedStats(ctx) - for _, allocStat := range bs.AllocationStats { - if allocStat.AllocationID == allocationID { - return allocStat, nil - } - } - return nil, common.NewError("allocation_stats_not_found", "Stats for allocation not found") - } - allocations := q.Get("allocations") - if allocations != "" { - return loadAllocationList(ctx) - } - bs.loadBasicStats(ctx) - return bs, nil -} diff --git a/code/go/0chain.net/blobbercore/writemarker/entity.go b/code/go/0chain.net/blobbercore/writemarker/entity.go index e99e5d01c..ebec8168d 100644 --- a/code/go/0chain.net/blobbercore/writemarker/entity.go +++ b/code/go/0chain.net/blobbercore/writemarker/entity.go @@ -52,6 +52,7 @@ type WriteMarkerEntity struct { CloseTxnID string `gorm:"column:close_txn_id;size:64"` ConnectionID string `gorm:"column:connection_id;size:64"` ClientPublicKey string `gorm:"column:client_key;size:256"` + Latest bool `gorm:"column:latest;not null;default:true"` Sequence int64 `gorm:"column:sequence;unique;autoIncrement;<-:false;index:idx_seq,unique,priority:2"` // <-:false skips value insert/update by gorm datastore.ModelWithTS } @@ -72,38 +73,47 @@ func (w *WriteMarkerEntity) BeforeSave(tx *gorm.DB) error { } func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, status WriteMarkerStatus, statusMessage, redeemTxn string) (err error) { - db := datastore.GetStore().GetTransaction(ctx) - statusBytes, _ := json.Marshal(statusMessage) + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + db := datastore.GetStore().GetTransaction(ctx) + statusBytes, _ := json.Marshal(statusMessage) + + if status == Failed { + wm.ReedeemRetries++ + err = db.Model(wm).Updates(WriteMarkerEntity{ + Status: status, + StatusMessage: string(statusBytes), + CloseTxnID: redeemTxn, + ReedeemRetries: wm.ReedeemRetries, + }).Error + return err + } - if status == Failed { - wm.ReedeemRetries++ err = db.Model(wm).Updates(WriteMarkerEntity{ - Status: status, - StatusMessage: string(statusBytes), - CloseTxnID: redeemTxn, - ReedeemRetries: wm.ReedeemRetries, + Status: status, + StatusMessage: string(statusBytes), + CloseTxnID: redeemTxn, }).Error - return - } - - err = db.Model(wm).Updates(WriteMarkerEntity{ - Status: status, - StatusMessage: string(statusBytes), - CloseTxnID: redeemTxn, - }).Error - if err != nil { - return - } - - // TODO (sfxdx): what about failed write markers ? - if status != Committed || wm.WM.Size <= 0 { - return // not committed or a deleting marker - } + if err != nil { + return err + } + + err = db.Exec("UPDATE write_markers SET latest = false WHERE allocation_id = ? AND allocation_root = ? AND sequence < ?", wm.WM.AllocationID, wm.WM.PreviousAllocationRoot, wm.Sequence).Error + if err != nil { + return err + } + + // TODO (sfxdx): what about failed write markers ? + if status != Committed || wm.WM.Size <= 0 { + return err // not committed or a deleting marker + } + + // work on pre-redeemed tokens and write-pools balances tracking + if err := allocation.AddToPending(ctx, wm.WM.ClientID, wm.WM.AllocationID, -wm.WM.Size); err != nil { + return fmt.Errorf("can't save allocation pending value: %v", err) + } + return nil + }) - // work on pre-redeemed tokens and write-pools balances tracking - if err := allocation.AddToPending(ctx, wm.WM.ClientID, wm.WM.AllocationID, -wm.WM.Size); err != nil { - return fmt.Errorf("can't save allocation pending value: %v", err) - } return } @@ -198,7 +208,7 @@ func (wm *WriteMarkerEntity) SendToChan(ctx context.Context) error { if sem == nil { sem = SetLock(wm.WM.AllocationID) } - err := sem.Acquire(ctx, 1) + err := sem.Acquire(context.TODO(), 1) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index fa8c2e707..a497aba88 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -2,16 +2,15 @@ package writemarker import ( "context" + "sync" "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" "go.uber.org/zap" - "gorm.io/gorm" ) // LockStatus lock status @@ -28,6 +27,11 @@ type LockResult struct { CreatedAt int64 `json:"created_at,omitempty"` } +var ( + lockPool = make(map[string]*WriteLock) + lockMutex sync.Mutex +) + // Mutex WriteMarker mutex type Mutex struct { // ML MapLocker @@ -51,45 +55,28 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string) (*L l, _ := m.ML.GetLock(allocationID) l.Lock() defer l.Unlock() - - db := datastore.GetStore().GetTransaction(ctx) - - var lock WriteLock - err := db.Table(TableNameWriteLock).Where("allocation_id=?", allocationID).First(&lock).Error - if err != nil { + lockMutex.Lock() + defer lockMutex.Unlock() + lock, ok := lockPool[allocationID] + if !ok { // new lock logging.Logger.Info("Creating new lock") - if errors.Is(err, gorm.ErrRecordNotFound) { - lock = WriteLock{ - AllocationID: allocationID, - ConnectionID: connectionID, - CreatedAt: time.Now(), - } - - err = db.Table(TableNameWriteLock).Create(&lock).Error - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } - - return &LockResult{ - Status: LockStatusOK, - CreatedAt: lock.CreatedAt.Unix(), - }, nil + lock = &WriteLock{ + CreatedAt: time.Now(), + ConnectionID: connectionID, } - logging.Logger.Error("Could not create lock") - //native postgres error - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) + lockPool[allocationID] = lock + return &LockResult{ + Status: LockStatusOK, + CreatedAt: lock.CreatedAt.Unix(), + }, nil } if lock.ConnectionID != connectionID { - if time.Since(lock.CreatedAt) > config.Configuration.WriteMarkerLockTimeout { + if time.Since(lock.CreatedAt) > config.Configuration.WriteMarkerLockTimeout || lock.ConnectionID == "" { // Lock expired. Provide lock to other connection id lock.ConnectionID = connectionID lock.CreatedAt = time.Now() - err = db.Model(&WriteLock{}).Where("allocation_id=?", allocationID).Save(&lock).Error - if err != nil { - return nil, errors.New("db_error", err.Error()) - } return &LockResult{ Status: LockStatusOK, CreatedAt: lock.CreatedAt.Unix(), @@ -103,10 +90,6 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string) (*L } lock.CreatedAt = time.Now() - err = db.Table(TableNameWriteLock).Where("allocation_id=?", allocationID).Save(&lock).Error - if err != nil { - return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) - } return &LockResult{ Status: LockStatusOK, @@ -118,16 +101,12 @@ func (*Mutex) Unlock(ctx context.Context, allocationID string, connectionID stri if allocationID == "" || connectionID == "" { return nil } - - db := datastore.GetStore().GetTransaction(ctx) - - err := db.Exec("DELETE FROM write_locks WHERE allocation_id = ? and connection_id = ? ", allocationID, connectionID).Error - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil - } - return errors.ThrowLog(err.Error(), common.ErrBadDataStore) + lockMutex.Lock() + defer lockMutex.Unlock() + lock, ok := lockPool[allocationID] + // reset lock if connection id matches + if ok && lock.ConnectionID == connectionID { + lock.ConnectionID = "" } - return nil } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 0c1532f4d..f053c0bad 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -9,7 +9,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - gomocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -56,16 +55,10 @@ func TestMutext_LockShouldWork(t *testing.T) { connectionID: "lock_same_connection_id", requestTime: now, mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_same_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_same_allocation_id", - "connection_id": "lock_same_connection_id", - "created_at": now, - }, - }) + lockPool["lock_same_allocation_id"] = &WriteLock{ + CreatedAt: now, + ConnectionID: "lock_same_connection_id", + } }, assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) @@ -79,16 +72,10 @@ func TestMutext_LockShouldWork(t *testing.T) { connectionID: "lock_pending_connection_id", requestTime: time.Now(), mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_allocation_id", - "connection_id": "lock_connection_id", - "created_at": time.Now().Add(-5 * time.Second), - }, - }) + lockPool["lock_allocation_id"] = &WriteLock{ + CreatedAt: time.Now().Add(-5 * time.Second), + ConnectionID: "lock_connection_id", + } }, assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) @@ -101,16 +88,10 @@ func TestMutext_LockShouldWork(t *testing.T) { connectionID: "lock_timeout_2nd_connection_id", requestTime: now, mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_timeout_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_timeout_allocation_id", - "connection_id": "lock_timeout_1st_connection_id", - "created_at": time.Now().Add(31 * time.Second), - }, - }) + lockPool["lock_timeout_allocation_id"] = &WriteLock{ + CreatedAt: time.Now().Add(31 * time.Second), + ConnectionID: "lock_timeout_1st_connection_id", + } }, assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) @@ -123,16 +104,10 @@ func TestMutext_LockShouldWork(t *testing.T) { connectionID: "lock_same_timeout_connection_id", requestTime: now, mock: func() { - gomocket.Catcher.NewMock(). - WithQuery(`SELECT * FROM "write_locks" WHERE allocation_id=$1 ORDER BY "write_locks"."allocation_id" LIMIT 1`). - WithArgs("lock_same_timeout_allocation_id"). - WithReply([]map[string]interface{}{ - { - "allocation_id": "lock_same_timeout_allocation_id", - "connection_id": "lock_same_timeout_connection_id", - "created_at": now.Add(-config.Configuration.WriteMarkerLockTimeout), - }, - }) + lockPool["lock_same_timeout_allocation_id"] = &WriteLock{ + CreatedAt: now.Add(-config.Configuration.WriteMarkerLockTimeout), + ConnectionID: "lock_same_timeout_connection_id", + } }, assert: func(test *testing.T, r *LockResult, err error) { require.Nil(test, err) diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index b0745aae0..007484f51 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -116,15 +116,14 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context) error { wme.Status = Committed wme.StatusMessage = t.TransactionOutput wme.CloseTxnID = t.Hash - err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) - return err + _ = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) + return nil } } txn, err := transaction.NewTransactionEntity() if err != nil { wme.StatusMessage = "Error creating transaction entity. " + err.Error() - wme.ReedeemRetries++ if err := wme.UpdateStatus(ctx, Failed, "Error creating transaction entity. "+err.Error(), ""); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } @@ -141,7 +140,6 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context) error { Logger.Error("Failed during sending close connection to the miner. ", zap.String("err:", err.Error())) wme.Status = Failed wme.StatusMessage = "Failed during sending close connection to the miner. " + err.Error() - wme.ReedeemRetries++ if err := wme.UpdateStatus(ctx, Failed, "Failed during sending close connection to the miner. "+err.Error(), ""); err != nil { Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) } @@ -154,7 +152,6 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context) error { Logger.Error("Error verifying the close connection transaction", zap.String("err:", err.Error()), zap.String("txn", txn.Hash)) wme.Status = Failed wme.StatusMessage = "Error verifying the close connection transaction." + err.Error() - wme.ReedeemRetries++ wme.CloseTxnID = txn.Hash // TODO Is this single try? if err := wme.UpdateStatus(ctx, Failed, "Error verifying the close connection transaction."+err.Error(), txn.Hash); err != nil { @@ -165,8 +162,8 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context) error { wme.Status = Committed wme.StatusMessage = t.TransactionOutput wme.CloseTxnID = t.Hash - err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) - return err + _ = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash) + return nil } func (wme *WriteMarkerEntity) VerifyRollbackMarker(ctx context.Context, dbAllocation *allocation.Allocation, latestWM *WriteMarkerEntity) error { diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 804449fc3..4f938ed3c 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -2,6 +2,7 @@ package writemarker import ( "context" + "strings" "sync" "time" @@ -19,6 +20,11 @@ var ( mut sync.RWMutex ) +// const ( +// timestampGap = 30 * 24 * 60 * 60 // 30 days +// cleanupWorkerInterval = 24 * 7 * time.Hour // 7 days +// ) + func SetupWorkers(ctx context.Context) { var res []allocation.Res @@ -38,6 +44,7 @@ func SetupWorkers(ctx context.Context) { } go startRedeem(ctx) + // go startCleanupWorker(ctx) } func GetLock(allocationID string) *semaphore.Weighted { @@ -78,6 +85,12 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { return err } + if alloc.Finalized { + logging.Logger.Info("Allocation is finalized. Skipping redeeming the write marker.", zap.Any("allocation", allocationID), zap.Any("wm", wm.WM.AllocationID)) + shouldRollback = true + return nil + } + if alloc.AllocationRoot != wm.WM.AllocationRoot { logging.Logger.Info("Stale write marker. Allocation root mismatch", zap.Any("allocation", allocationID), @@ -97,7 +110,9 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { logging.Logger.Error("Error redeeming the write marker.", zap.Any("allocation", allocationID), zap.Any("wm", wm), zap.Any("error", err), zap.Any("elapsedTime", elapsedTime)) - go tryAgain(wm) + if retryRedeem(err.Error()) { + go tryAgain(wm) + } shouldRollback = true return err @@ -108,7 +123,7 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { mut.Release(1) } }() - err = allocation.Repo.UpdateAllocationRedeem(ctx, wm.WM.AllocationRoot, allocationID, alloc) + err = allocation.Repo.UpdateAllocationRedeem(ctx, allocationID, wm.WM.AllocationRoot, alloc) if err != nil { logging.Logger.Error("Error redeeming the write marker. Allocation latest wm redeemed update failed", zap.Any("allocation", allocationID), @@ -165,5 +180,31 @@ func startRedeem(ctx context.Context) { } func tryAgain(wm *WriteMarkerEntity) { + time.Sleep(time.Duration(wm.ReedeemRetries) * 5 * time.Second) writeMarkerChan <- wm } + +// Can add more cases where we don't want to retry +func retryRedeem(errString string) bool { + return !strings.Contains(errString, "value not present") +} + +// TODO: don't delete prev WM +// func startCleanupWorker(ctx context.Context) { +// for { +// select { +// case <-ctx.Done(): +// return +// case <-time.After(cleanupWorkerInterval): +// _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { +// tx := datastore.GetStore().GetTransaction(ctx) +// timestamp := int64(common.Now()) - timestampGap // 30 days +// err := tx.Exec("INSERT INTO write_markers_archive (SELECT * from write_markers WHERE timestamp < ? AND latest = )", timestamp, false).Error +// if err != nil { +// return err +// } +// return tx.Exec("DELETE FROM write_markers WHERE timestamp < ? AND latest = )", timestamp, false).Error +// }) +// } +// } +// } diff --git a/code/go/0chain.net/blobbercore/writemarker/write_lock.go b/code/go/0chain.net/blobbercore/writemarker/write_lock.go index f5790bea4..a3c3cd11f 100644 --- a/code/go/0chain.net/blobbercore/writemarker/write_lock.go +++ b/code/go/0chain.net/blobbercore/writemarker/write_lock.go @@ -2,18 +2,8 @@ package writemarker import "time" -const ( - TableNameWriteLock = "write_locks" -) - // WriteLock WriteMarker lock type WriteLock struct { - AllocationID string `gorm:"column:allocation_id;size:64;primaryKey"` - ConnectionID string `gorm:"column:connection_id;size:64"` - CreatedAt time.Time `gorm:"column:created_at"` -} - -// TableName get table name of migrate -func (WriteLock) TableName() string { - return TableNameWriteLock + ConnectionID string + CreatedAt time.Time } diff --git a/code/go/0chain.net/conductor/conductrpc/client.go b/code/go/0chain.net/conductor/conductrpc/client.go index e73885538..5ef0afff2 100644 --- a/code/go/0chain.net/conductor/conductrpc/client.go +++ b/code/go/0chain.net/conductor/conductrpc/client.go @@ -52,3 +52,13 @@ func (c *client) blobberCommitted(blobberID string) (err error) { err = c.client.Call("Server.BlobberCommitted", blobberID, nil) return } + +func (c *client) validationTicketGenerated(ticket ValidtorTicket) (err error) { + err = c.client.Call("Server.ValidatorTicket", &ticket, nil) + return +} + +func (c *client) sendFileMetaRoot(m map[string]string) (err error) { + err = c.client.Call("Server.GetFileMetaRoot", m, nil) + return +} diff --git a/code/go/0chain.net/conductor/conductrpc/entity.go b/code/go/0chain.net/conductor/conductrpc/entity.go index f1cc05ac4..c52db9bf3 100644 --- a/code/go/0chain.net/conductor/conductrpc/entity.go +++ b/code/go/0chain.net/conductor/conductrpc/entity.go @@ -1,6 +1,7 @@ package conductrpc import ( + "context" "log" "sync" "sync/atomic" @@ -31,6 +32,10 @@ func (e *Entity) State() (state *State) { // SetState sets current state. func (e *Entity) SetState(state *State) { e.state.Store(state) // update + + if state.GetFileMetaRoot { + go SendFileMetaRoot() + } } // NewEntity creates RPC client for integration tests. @@ -107,6 +112,25 @@ func (e *Entity) BlobberCommitted(blobberID string) { } } +func (e *Entity) ValidatorTicket(ticket ValidtorTicket) { + err := e.client.validationTicketGenerated(ticket) + if err != nil { + log.Println(err) + } +} + +func (e *Entity) SendFileMetaRoot(blobberID, fileMetaRoot string, ctxCncl context.CancelFunc) { + m := map[string]string{ + "blobber_id": blobberID, + "file_meta_root": fileMetaRoot, + } + err := e.client.sendFileMetaRoot(m) + if err != nil { + return + } + ctxCncl() +} + // // global // diff --git a/code/go/0chain.net/conductor/conductrpc/file_meta.go b/code/go/0chain.net/conductor/conductrpc/file_meta.go new file mode 100644 index 000000000..b203c5ec6 --- /dev/null +++ b/code/go/0chain.net/conductor/conductrpc/file_meta.go @@ -0,0 +1,64 @@ +package conductrpc + +import ( + "context" + + "log" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + + "github.com/0chain/blobber/code/go/0chain.net/core/node" +) + +// alreadyRunning is simple indicator that given function is running +// no need to acquire mutex lock. It does not matter if at a time it +// somehow runs the given function multiple times. Since it takes some +// time to acquire state from rpc server there is no concurrent running +var alreadyRunning bool + +func SendFileMetaRoot() { + if alreadyRunning { + return + } + alreadyRunning = true + defer func() { + alreadyRunning = false + }() + + ctx, ctxCncl := context.WithCancel(context.TODO()) + defer ctxCncl() + + for { + select { + case <-ctx.Done(): + return + default: + } + + s := global.State() + if s.GetFileMetaRoot { + fmr, err := getFileMetaRoot() + if err != nil { + log.Printf("Error: %v", err) + continue + } + + global.SendFileMetaRoot(node.Self.ID, fmr, ctxCncl) + } + } +} + +func getFileMetaRoot() (string, error) { + db := datastore.GetStore().GetDB() + var fmr string + + // It will work fine because this blobber will have only single allocation + // created by conductor + err := db.Raw("SELECT file_meta_root FROM allocations LIMIT 1").Scan(&fmr).Error + + if err != nil { + return "", err + } + + return fmr, nil +} diff --git a/code/go/0chain.net/conductor/conductrpc/server.go b/code/go/0chain.net/conductor/conductrpc/server.go index 01979c52a..00c23fc14 100644 --- a/code/go/0chain.net/conductor/conductrpc/server.go +++ b/code/go/0chain.net/conductor/conductrpc/server.go @@ -16,3 +16,7 @@ type ( RoundName = config.RoundName Number = config.Number ) + +type ValidtorTicket struct { + ValidatorId string +} diff --git a/code/go/0chain.net/conductor/conductrpc/state.go b/code/go/0chain.net/conductor/conductrpc/state.go index d244a4a44..5d5da96a6 100644 --- a/code/go/0chain.net/conductor/conductrpc/state.go +++ b/code/go/0chain.net/conductor/conductrpc/state.go @@ -62,7 +62,12 @@ type State struct { BlobberUpload BlobberUpload BlobberDelete BlobberDelete AdversarialValidator AdversarialValidator + NotifyOnValidationTicketGeneration bool StopWMCommit *bool + FailRenameCommit []string + FailUploadCommit []string + GetFileMetaRoot bool + MissUpDownload bool } // Name returns NodeName by given NodeID. diff --git a/code/go/0chain.net/core/common/admin.go b/code/go/0chain.net/core/common/admin.go index 5cd87800e..d43a75fbf 100644 --- a/code/go/0chain.net/core/common/admin.go +++ b/code/go/0chain.net/core/common/admin.go @@ -8,25 +8,29 @@ import ( // global username and password used to access endpoints only by admin var gUsername, gPassword string +var isDevelopment bool -func SetAdminCredentials() { +func SetAdminCredentials(devMode bool) { gUsername = viper.GetString("admin.username") gPassword = viper.GetString("admin.password") + isDevelopment = devMode } func AuthenticateAdmin(handler ReqRespHandlerf) ReqRespHandlerf { return func(w http.ResponseWriter, r *http.Request) { - uname, passwd, ok := r.BasicAuth() - if !ok { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Admin only api")) // nolint - return - } + if !isDevelopment { + uname, passwd, ok := r.BasicAuth() + if !ok { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Admin only api")) // nolint + return + } - if uname != gUsername || passwd != gPassword { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid username or password")) // nolint - return + if uname != gUsername || passwd != gPassword { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Invalid username or password")) // nolint + return + } } handler(w, r) diff --git a/code/go/0chain.net/core/common/aws.go b/code/go/0chain.net/core/common/aws.go new file mode 100644 index 000000000..e82dfda8e --- /dev/null +++ b/code/go/0chain.net/core/common/aws.go @@ -0,0 +1,32 @@ +package common + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" +) + +func GetSecretsFromAWS(secretName, region string) (string, error) { + c, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) + if err != nil { + return "", err + } + + // Create Secrets Manager client + svc := secretsmanager.NewFromConfig(c) + + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretName), + VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified + } + + result, err := svc.GetSecretValue(context.TODO(), input) + if err != nil { + return "", err + } + + secretString := *result.SecretString + return secretString, nil +} \ No newline at end of file diff --git a/code/go/0chain.net/core/common/constants.go b/code/go/0chain.net/core/common/constants.go index 9964b3a5f..29a6a9130 100644 --- a/code/go/0chain.net/core/common/constants.go +++ b/code/go/0chain.net/core/common/constants.go @@ -43,4 +43,7 @@ var ( // ErrNotFound ref is not found ErrNotFound = errors.New("ref is not found") + + // ErrNoThumbnail no thumbnail + ErrNoThumbnail = errors.New("no thumbnail") ) diff --git a/code/go/0chain.net/core/common/handler.go b/code/go/0chain.net/core/common/handler.go index 27ca8585c..d4de9afb8 100644 --- a/code/go/0chain.net/core/common/handler.go +++ b/code/go/0chain.net/core/common/handler.go @@ -82,7 +82,12 @@ func ToByteStream(handler JSONResponderF) ReqRespHandlerf { w.Write(rawdata) //nolint:errcheck } else { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(data) //nolint:errcheck + byteData, err := json.Marshal(data) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + w.Write(byteData) //nolint:errcheck } } } diff --git a/code/go/0chain.net/core/common/request_form.go b/code/go/0chain.net/core/common/request_form.go index c5d91a867..4bdfe933a 100644 --- a/code/go/0chain.net/core/common/request_form.go +++ b/code/go/0chain.net/core/common/request_form.go @@ -3,7 +3,7 @@ package common import "net/http" const ( - FormFileParseMaxMemory = 10 * 1024 * 1024 + FormFileParseMaxMemory = 32 * 1024 * 1024 ) // TryParseForm try populates r.Form and r.PostForm. diff --git a/code/go/0chain.net/core/encryption/hash.go b/code/go/0chain.net/core/encryption/hash.go index 0b098fe00..53f649343 100644 --- a/code/go/0chain.net/core/encryption/hash.go +++ b/code/go/0chain.net/core/encryption/hash.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "encoding/hex" + "github.com/minio/sha256-simd" "golang.org/x/crypto/sha3" ) @@ -34,6 +35,23 @@ func RawHash(data interface{}) []byte { return hash.Sum(nil) } +func ShaHash(data interface{}) []byte { + var databuf []byte + switch dataImpl := data.(type) { + case []byte: + databuf = dataImpl + case HashBytes: + databuf = dataImpl[:] + case string: + databuf = []byte(dataImpl) + default: + panic("unknown type") + } + hash := sha256.New() + _, _ = hash.Write(databuf) + return hash.Sum(nil) +} + /*FastHash - sha1 hash the given data and return the hash as hex string */ func FastHash(data interface{}) string { return hex.EncodeToString(RawFastHash(data)) diff --git a/code/go/0chain.net/core/lock/lock.go b/code/go/0chain.net/core/lock/lock.go index ea026939a..ba258be03 100644 --- a/code/go/0chain.net/core/lock/lock.go +++ b/code/go/0chain.net/core/lock/lock.go @@ -33,6 +33,9 @@ func (m *Mutex) RLock() { } func (m *Mutex) RUnlock() { + lockMutex.Lock() + defer lockMutex.Unlock() + m.usedby-- m.mu.RUnlock() } diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index c2bd8837c..4e0a9183e 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -2,10 +2,10 @@ package transaction import ( "encoding/json" + "github.com/0chain/gosdk/core/transaction" "sync" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" @@ -16,6 +16,11 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/node" ) +var ( + Last50Transactions []string + last50TransactionsMutex sync.Mutex +) + // Transaction entity that encapsulates the transaction related data and meta data type Transaction struct { Hash string `json:"hash,omitempty"` @@ -92,10 +97,6 @@ type StorageAllocation struct { ParityShards int64 `json:"parity_shards"` } -func (sa *StorageAllocation) Until() common.Timestamp { - return sa.Expiration + common.Timestamp(config.StorageSCConfig.ChallengeCompletionTime/time.Second) -} - type StorageAllocationBlobber struct { BlobberID string `json:"blobber_id"` Size int64 `json:"size"` @@ -139,6 +140,15 @@ func (t *Transaction) GetTransaction() zcncore.TransactionScheme { func (t *Transaction) ExecuteSmartContract(address, methodName string, input interface{}, val uint64) error { t.wg.Add(1) + + sn := transaction.SmartContractTxnData{Name: methodName, InputArgs: input} + snBytes, err := json.Marshal(sn) + if err != nil { + return err + } + + updateLast50Transactions(string(snBytes)) + nonce := monitor.getNextUnusedNonce() if err := t.zcntxn.SetTransactionNonce(nonce); err != nil { logging.Logger.Error("Failed to set nonce.", @@ -151,7 +161,7 @@ func (t *Transaction) ExecuteSmartContract(address, methodName string, input int zap.Any("hash", t.zcntxn.GetTransactionHash()), zap.Any("nonce", nonce)) - _, err := t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val)) + _, err = t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val)) if err != nil { t.wg.Done() logging.Logger.Error("Failed to execute SC.", @@ -260,3 +270,14 @@ func (t *Transaction) OnVerifyComplete(zcntxn *zcncore.Transaction, status int) func (t *Transaction) OnAuthComplete(zcntxn *zcncore.Transaction, status int) { } + +func updateLast50Transactions(data string) { + last50TransactionsMutex.Lock() + defer last50TransactionsMutex.Unlock() + + if len(Last50Transactions) == 50 { + Last50Transactions = Last50Transactions[1:] + } else { + Last50Transactions = append(Last50Transactions, data) + } +} diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 349f9d2bc..52feee60c 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -1,23 +1,9 @@ package transaction import ( - "bytes" - "context" - "hash/fnv" - "math" - - "fmt" - "io" - "net/http" - "net/url" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - - "github.com/0chain/errors" - "github.com/0chain/gosdk/core/resty" - "github.com/0chain/gosdk/core/util" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk/zboxcore/zboxutil" ) const TXN_SUBMIT_URL = "v1/transaction/put" @@ -30,9 +16,11 @@ const ( ) var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders") -var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) = makeSCRestAPICall +var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string) ([]byte, error) = MakeSCRestAPICallNoHandler -type SCRestAPIHandler func(response map[string][]byte, numSharders int, err error) +func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string) ([]byte, error) { + return zboxutil.MakeSCRestAPICall(address, path, params, nil) +} func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) { txn, err := NewTransactionEntity() @@ -64,120 +52,3 @@ func VerifyTransactionWithNonce(txnHash string, nonce int64) (*Transaction, erro } return txn, nil } - -// MakeSCRestAPICall execute api reqeust from sharders, and parse and return result -func makeSCRestAPICall(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { - var resMaxCounterBody []byte - - var hashMaxCounter int - - network := zcncore.GetNetwork() - numSharders := len(network.Sharders) - - if numSharders == 0 { - return nil, ErrNoAvailableSharder - } - - minNumConfirmation := int(math.Ceil(float64(MinConfirmation*numSharders) / 100)) - - rand := util.NewRand(numSharders) - - selectedSharders := make([]string, 0, minNumConfirmation+1) - - // random pick minNumConfirmation+1 first - for i := 0; i <= minNumConfirmation; i++ { - n, err := rand.Next() - - if err != nil { - break - } - selectedSharders = append(selectedSharders, network.Sharders[n]) - } - - urls := make([]string, 0, len(network.Sharders)) - - q := url.Values{} - for k, v := range params { - q.Add(k, v) - } - - for _, sharder := range selectedSharders { - u := fmt.Sprintf("%v/%v%v%v", sharder, SC_REST_API_URL, scAddress, relativePath) - - urls = append(urls, u+"?"+q.Encode()) - } - - header := map[string]string{ - "Content-Type": "application/json; charset=utf-8", - "Access-Control-Allow-Origin": "*", - } - - //leave first item for ErrTooLessConfirmation - var msgList = make([]string, 1, numSharders) - - r := resty.New(resty.WithHeader(header)).Then(func(req *http.Request, resp *http.Response, respBody []byte, cancelFunc context.CancelFunc, err error) error { - if err != nil { //network issue - msgList = append(msgList, err.Error()) - return err - } - - url := req.URL.String() - - if resp.StatusCode != http.StatusOK { - errorMsg := "[sharder]" + resp.Status + ": " + url - msgList = append(msgList, errorMsg) - - return errors.Throw(ErrBadRequest, errorMsg) - } - - hash := fnv.New32() //use fnv for better performance - - teeReader := io.TeeReader(bytes.NewReader(respBody), hash) - resBody, err := io.ReadAll(teeReader) - - if err != nil { - errorMsg := "[sharder]body: " + url + " " + err.Error() - msgList = append(msgList, errorMsg) - return errors.Throw(ErrBadRequest, errorMsg) - } - - // NOTE: This would only return the last response, and no consensus is - // actually met. This can be a workaround for the fix. But actually - // it's hard to have consensus as the sharders could be in different - // LFB when they receive the request, which means they would give - // different response. - resMaxCounterBody = resBody - hashMaxCounter++ - - return nil - }) - - for { - r.DoGet(context.TODO(), urls...) - - r.Wait() - - if hashMaxCounter >= minNumConfirmation { - break - } - - // pick more one sharder to query transaction - n, err := rand.Next() - - if errors.Is(err, util.ErrNoItem) { - break - } - - urls = []string{ - fmt.Sprintf("%v/%v%v%v", network.Sharders[n], SC_REST_API_URL, scAddress, relativePath) + "?" + q.Encode(), - } - } - - if hashMaxCounter < minNumConfirmation { - msgList[0] = fmt.Sprintf("min_confirmation is %v%%, but got %v/%v sharders", MinConfirmation, hashMaxCounter, numSharders) - - return nil, errors.Throw(ErrTooLessConfirmation, msgList...) - } - - return resMaxCounterBody, nil -} diff --git a/code/go/0chain.net/swagger.md b/code/go/0chain.net/swagger.md index a9f0f1028..cc44c5446 100644 --- a/code/go/0chain.net/swagger.md +++ b/code/go/0chain.net/swagger.md @@ -12,7 +12,6 @@ ## Content negotiation - ### URI Schemes * http * https diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index 72bcf5c24..906cf64d2 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -8,6 +8,7 @@ import ( "os" "runtime" "strconv" + "strings" "time" "github.com/0chain/blobber/code/go/0chain.net/core/build" @@ -40,6 +41,8 @@ func initHandlers(r *mux.Router) { storage.SetupHandlers(r) } +var publicKey, privateKey string + func main() { fmt.Println("======[ Validator ]======") @@ -90,13 +93,17 @@ func main() { fmt.Printf("[+] %-24s %s\n", "setup configs", "[OK]") - // register on chain - reader, err := os.Open(*keysFile) + err := readKeysFromAws() if err != nil { - panic(err) + err = readKeysFromFile(keysFile) + if err != nil { + panic(err) + } + fmt.Println("using validator keys from local") + } else { + fmt.Println("using validator keys from aws") } - publicKey, privateKey, _, _ := encryption.ReadKeys(reader) - reader.Close() + node.Self.SetKeys(publicKey, privateKey) if len(*hostUrl) > 0 { @@ -122,7 +129,7 @@ func main() { Logger.Info(" Base URL" + node.Self.GetURLBase()) - prepare("validator-" + node.Self.ID) + prepare(node.Self.ID) config.SetServerChainID(config.Configuration.ChainID) common.SetupRootContext(node.GetNodeContext()) @@ -259,3 +266,29 @@ func HomePageHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%v\n", sharder) } } + +func readKeysFromAws() error { + blobberSecretName := os.Getenv("VALIDATOR_SECRET_NAME") + awsRegion := os.Getenv("AWS_REGION") + keys, err := common.GetSecretsFromAWS(blobberSecretName, awsRegion) + if err != nil { + return err + } + secretsFromAws := strings.Split(keys, "\n") + if len(secretsFromAws) < 2 { + return fmt.Errorf("wrong file format from aws") + } + publicKey = secretsFromAws[0] + privateKey = secretsFromAws[1] + return nil +} + +func readKeysFromFile(keysFile *string) error { + reader, err := os.Open(*keysFile) + if err != nil { + return err + } + defer reader.Close() + publicKey, privateKey, _, _ = encryption.ReadKeys(reader) + return nil +} \ No newline at end of file diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go b/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go index 26e8705fc..04e782ca8 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go @@ -49,5 +49,11 @@ func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error, res, err := challengeHandler(ctx, r) + if state.NotifyOnValidationTicketGeneration { + conductrpc.Client().ValidatorTicket(conductrpc.ValidtorTicket{ + ValidatorId: node.Self.ID, + }) + } + return res, err, true } diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go b/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go index 27480089d..b353a5f12 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go @@ -9,5 +9,12 @@ import ( ) func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error) { - return challengeHandler(ctx, r) + res, err := challengeHandler(ctx, r) + + if len(Last5Transactions) >= 5 { + Last5Transactions = Last5Transactions[1:] + } + Last5Transactions = append(Last5Transactions, res) + + return res, err } diff --git a/code/go/0chain.net/validatorcore/storage/handler_main.go b/code/go/0chain.net/validatorcore/storage/handler_main.go index 82b46eb0c..7af0dbbb2 100644 --- a/code/go/0chain.net/validatorcore/storage/handler_main.go +++ b/code/go/0chain.net/validatorcore/storage/handler_main.go @@ -19,5 +19,5 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/debug", common.ToJSONResponse(DumpGoRoutines)) - r.HandleFunc("/stats", statsHandler) + r.HandleFunc("/_stats", statsHandler) } diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index a7dbe543e..ba47ef2f8 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -330,10 +330,10 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO latestWM := cr.WriteMarkers[len(cr.WriteMarkers)-1].WM if cr.ObjPath != nil { rootRef := cr.ObjPath.RootObject - allocationRootCalculated := rootRef.Hash + allocationRootCalculated := cr.ObjPath.RootHash if latestWM.AllocationRoot != allocationRootCalculated { - return common.NewError("challenge_validation_failed", "Allocation root does not match") + return common.NewError("challenge_validation_failed", fmt.Sprintf("Allocation root does not match expected %s got %s", allocationRootCalculated, latestWM.AllocationRoot)) } if rootRef.NumBlocks == 0 { @@ -356,7 +356,7 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO } logging.Logger.Info("Verifying data block and merkle path", zap.String("challenge_id", challengeObj.ID)) - fHash := encryption.RawHash(cr.ChallengeProof.Data) + fHash := encryption.ShaHash(cr.ChallengeProof.Data) fixedMerkleRoot, _ := hex.DecodeString(cr.ObjPath.Meta.FixedMerkleRoot) fmp := &util.FixedMerklePath{ LeafHash: fHash, diff --git a/code/go/0chain.net/validatorcore/storage/protocol.go b/code/go/0chain.net/validatorcore/storage/protocol.go index 058025843..521d3ee01 100644 --- a/code/go/0chain.net/validatorcore/storage/protocol.go +++ b/code/go/0chain.net/validatorcore/storage/protocol.go @@ -88,7 +88,7 @@ func (sp *ValidatorProtocolImpl) VerifyChallengeTransaction(ctx context.Context, params := make(map[string]string) params["blobber"] = blobberID params["challenge"] = challengeRequest.ChallengeID - challengeBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/getchallenge", params, chain.GetServerChain()) + challengeBytes, err := transaction.MakeSCRestAPICall(transaction.STORAGE_CONTRACT_ADDRESS, "/getchallenge", params) if err != nil { return nil, common.NewError("invalid_challenge", "Invalid challenge id. Challenge not found in blockchain. "+err.Error()) diff --git a/code/go/0chain.net/validatorcore/storage/stats_handler.go b/code/go/0chain.net/validatorcore/storage/stats_handler.go index 94ee177d6..a0326acf0 100644 --- a/code/go/0chain.net/validatorcore/storage/stats_handler.go +++ b/code/go/0chain.net/validatorcore/storage/stats_handler.go @@ -1,11 +1,14 @@ package storage import ( + "encoding/json" "fmt" "net/http" "sync" ) +var Last5Transactions []interface{} + type Stats struct { TotalChallenges int SuccessfulChallenges int @@ -88,9 +91,24 @@ func statsHandler(w http.ResponseWriter, r *http.Request) { ` + fmt.Sprintf("%d", result.FailedChallenges) + ` - - - ` +
+

Last 5 Transactions

+
    + ` + for _, transaction := range Last5Transactions { + jsonData, err := json.Marshal(transaction) + if err != nil { + statsHTML += "
  • Failed to marshal transaction
    • " + continue + } + statsHTML += "
  • " + string(jsonData) + "
    • " + } + statsHTML += ` +
+
+ + + ` w.Header().Set("Content-Type", "text/html") _, err := w.Write([]byte(statsHTML)) diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index b4c845b25..94e5ef1df 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -12,12 +12,15 @@ logging: # 100 GB - 107374182400 capacity: 1073741824 # 1 GB bytes total blobber capacity read_price: 0.00 # token / GB for reading -write_price: 0.10 # token / GB / time_unit for writing +write_price: 0.025 # token / GB / time_unit for writing price_in_usd: false price_worker_in_hours: 12 # update_allocations_interval used to refresh known allocation objects from SC -update_allocations_interval: 1m +update_allocations_interval: 60m + +#finalize_allocations_interval used to get and finalize empty allocations +finalize_allocations_interval: 24h # maximum limit on the number of combined directories and files on each allocation max_dirs_files: 50000 @@ -25,13 +28,13 @@ max_dirs_files: 50000 # delegate wallet (must be set) delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9' # maximum allowed number of stake holders -num_delegates: 50 +num_delegates: 10 # service charge of the blobber -service_charge: 0.10 +service_charge: 0 # min submit from miners` -min_submit: 50 +min_submit: 20 # min confirmation from sharder -min_confirmation: 50 +min_confirmation: 10 block_worker: https://dev.0chain.net/dns @@ -59,7 +62,16 @@ rate_limiters: # General Request Per Second. This rps is used to rate limit endpoints like copy, rename, get file metadata, # get paginated refs, etc. Default is 5 general_rps: 1600 - + # Number of blocks downloaded in a day. Default is 100GB(the value needs to be in blocks which is data/64KB) + block_limit_daily: 1562500 + # Max blocks per download request. Default is 500 + block_limit_request: 500 + # Max blocks in a month for a client. Default is 2000GB(the value needs to be in blocks which is data/64KB) + block_limit_monthly: 31250000 + # Max upload limit in a month for a client. Default is 2000GB(the value needs to be in blocks which is data/64KB) + upload_limit_monthly: 31250000 + # Max commit limit in a month for a client. Default is 30000 + commit_limit_monthly: 30000 server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" owner: "edb90b850f2e7e7cbd0a1fa370fdcc5cd378ffbec95363a7bc0e5a98b8ba5759" @@ -83,6 +95,7 @@ challenge_response: frequency: 10 num_workers: 5 max_retries: 20 + cleanup_gap: 100000 healthcheck: frequency: 60m # send healthcheck to miners every 60 minutes @@ -96,6 +109,7 @@ db: password: blobber host: postgres port: 5432 + archive_path: "/var/lib/postgresql/hdd" storage: files_dir: "/path/to/hdd" @@ -107,9 +121,10 @@ storage: # Similarly for some file_hash "ef935503b66b1ce026610edf18bffd756a79676a8fe317d951965b77a77c0227" with dir_level [2, 2, 1] # following path is created for the file: # {alloc_dir}/ef/93/5/503b66b1ce026610edf18bffd756a79676a8fe317d951965b77a77c0227 - alloc_dir_level: [2, 1] + alloc_dir_level: [2, 1] #TODO sort it out file_dir_level: [2, 2, 1] +#TODO remove disk_update: # defaults to true. If false, blobber has to manually update blobber's capacity upon increase/decrease # If blobber has to limit its capacity to 5% of its capacity then it should turn automaci_update to false. diff --git a/config/0chain_validator.yaml b/config/0chain_validator.yaml index ae84bf9d0..0c8d91322 100644 --- a/config/0chain_validator.yaml +++ b/config/0chain_validator.yaml @@ -3,9 +3,9 @@ version: 1.0 # delegate wallet (must be set) delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9' # maximum allowed number of stake holders -num_delegates: 50 +num_delegates: 1 # service charge of related blobber -service_charge: 0.10 +service_charge: 1 #TODO CHECK IF 0 WORKS WELL block_worker: https://dev.0chain.net/dns @@ -21,7 +21,7 @@ logging: console: true # printing log to console is only supported in development mode healthcheck: - frequency: 60s # send healthcheck to miners every 60 seconds + frequency: 50m # send healthcheck to miners every 60 seconds server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index 05bc5aa32..0f6b48fb9 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -1,23 +1,38 @@ version: "3" services: postgres: + container_name: postgres-blob-${BLOBBER} image: postgres:14 environment: + POSTGRES_DB: blobber_meta POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: secret + POSTGRES_HOST: postgres-blob-${BLOBBER} + POSTGRES_USER: blobber_user + POSTGRES_PASSWORD: blobber POSTGRES_HOST_AUTH_METHOD: trust -#expose_ci_port + SLOW_TABLESPACE_PATH: /var/lib/postgresql/hdd # this should match with archive_path in 0chain_blobber.yaml + SLOW_TABLESPACE: hdd_tablespace # this should match with the dbs.events.slowtablespace in 0chain.yaml volumes: + - ../config/postgresql.conf:/etc/postgresql/postgresql.conf - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data - ./sql_init:/docker-entrypoint-initdb.d + # - ../blobber${BLOBBER}/data/postgresql2:/var/lib/postgresql/hdd + command: postgres -c config_file=/etc/postgresql/postgresql.conf + restart: unless-stopped networks: default: + validator: + container_name: validator-${BLOBBER} image: validator environment: - - DOCKER= true + - DOCKER=true + - AWS_ACCESS_KEY_ID=key_id + - AWS_SECRET_ACCESS_KEY=secret_key + - VALIDATOR_SECRET_NAME=validator_secret_name + - AWS_REGION=aws_region + depends_on: + - postgres volumes: - ${CONFIG_PATH:-../config}:/validator/config #value after :- is default value - ./blobber${BLOBBER}/data:/validator/data @@ -26,19 +41,32 @@ services: ports: - "506${BLOBBER}:506${BLOBBER}" command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0vnode${BLOBBER}_keys.txt --log_dir /validator/log + restart: unless-stopped networks: default: testnet0: ipv4_address: 198.18.0.6${BLOBBER} blobber: + container_name: blobber-${BLOBBER} image: blobber environment: - - DOCKER= true + - DOCKER=true + - DB_HOST=postgres-blob-${BLOBBER} + - DB_NAME=blobber_meta + - DB_USER=blobber_user + - DB_PASSWORD=blobber + - DB_PORT=5432 + - AWS_ACCESS_KEY_ID=key_id + - AWS_SECRET_ACCESS_KEY=secret_key + - BLOBBER_SECRET_NAME=blobber_secret_name + - AWS_REGION=aws_region depends_on: - validator + - postgres links: - validator:validator + - postgres:postgres volumes: - ${CONFIG_PATH:-../config}:/blobber/config - ./blobber${BLOBBER}/files:/blobber/files @@ -50,6 +78,7 @@ services: - "505${BLOBBER}:505${BLOBBER}" - "3150${BLOBBER}:3150${BLOBBER}" command: ./bin/blobber --port 505${BLOBBER} --grpc_port 3150${BLOBBER} --hostname 198.18.0.9${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data + restart: unless-stopped networks: default: testnet0: diff --git a/docker.local/conductor-config/0chain_blobber.yaml b/docker.local/conductor-config/0chain_blobber.yaml index 987cdd1cb..85fc52664 100644 --- a/docker.local/conductor-config/0chain_blobber.yaml +++ b/docker.local/conductor-config/0chain_blobber.yaml @@ -1,7 +1,7 @@ version: "1.0" logging: - level: "info" + level: "debug" console: true # printing log to console is only supported in development mode info: @@ -35,6 +35,9 @@ write_lock_timeout: 1m # update_allocations_interval used to refresh known allocation objects from SC update_allocations_interval: 1m +#finalize_allocations_interval used to get and finalize empty allocations +finalize_allocations_interval: 24h + # maximum limit on the number of combined directories and files on each allocation max_dirs_files: 50000 diff --git a/docker.local/conductor-config/0chain_validator.yaml b/docker.local/conductor-config/0chain_validator.yaml index abc25d78b..35883065e 100644 --- a/docker.local/conductor-config/0chain_validator.yaml +++ b/docker.local/conductor-config/0chain_validator.yaml @@ -17,7 +17,7 @@ rate_limiters: proxy: true logging: - level: "error" + level: "debug" console: true # printing log to console is only supported in development mode healthcheck: diff --git a/docker.local/sql_init/000-init-db.sh b/docker.local/sql_init/000-init-db.sh new file mode 100755 index 000000000..3e2c20d6e --- /dev/null +++ b/docker.local/sql_init/000-init-db.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +mkdir -p $SLOW_TABLESPACE_PATH + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL +create tablespace $SLOW_TABLESPACE location '$SLOW_TABLESPACE_PATH'; +EOSQL diff --git a/go.mod b/go.mod index faf9a96ea..f557b26a9 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.18 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 + github.com/0chain/gosdk v1.11.0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 github.com/gorilla/handlers v1.5.1 - github.com/gorilla/mux v1.8.0 + github.com/gorilla/mux v1.8.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 github.com/herumi/bls-go-binary v1.31.0 @@ -21,9 +21,9 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.15.0 golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.14.0 golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect google.golang.org/grpc v1.56.2 @@ -33,7 +33,7 @@ require ( gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2 + gorm.io/gorm v1.25.5 ) require ( @@ -44,10 +44,30 @@ require ( require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/minio/sha256-simd v1.0.1 google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect ) -require github.com/pressly/goose/v3 v3.13.4 +require ( + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5 + github.com/pressly/goose/v3 v3.13.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect +) require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect @@ -65,7 +85,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.11.4 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.21.4 // indirect @@ -85,6 +105,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/h2non/filetype v1.1.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -94,7 +115,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/klauspost/reedsolomon v1.11.7 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/machinebox/graphql v0.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -123,7 +144,7 @@ require ( go.mongodb.org/mongo-driver v1.11.3 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 25c8fad34..237c28101 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 h1:GHTdYTmhNY9genBkNWLXdn3Z1yCtcbSNkcIFaKrqBRU= -github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= +github.com/0chain/gosdk v1.11.0 h1:PSD4ohQaaSOsH/sHvfnCbq35Bs5fCtL1g9S4vyvxQOY= +github.com/0chain/gosdk v1.11.0/go.mod h1:DAg/de6vodjEa7CM1/LjElOwntRtNV5lb9rMRaR7fzU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -75,6 +75,34 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5 h1:qYi/BfDrWXZxlmRjlKCyFmtI4HKJwW8OKDKhKRAOZQI= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5/go.mod h1:4Ae1NCLK6ghmjzd45Tc33GgCKhUWD2ORAlULtMO1Cbs= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -334,8 +362,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -379,8 +407,8 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -414,6 +442,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -423,7 +453,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wog6OhmI= github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= @@ -479,8 +509,8 @@ github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGC github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= -github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20 h1:R7RAW1p8wjhlHKFhS4X7h8EePqADev/PltCmW9qlJoM= github.com/koding/cache v0.0.0-20161222233015-e8a81b0b3f20/go.mod h1:sh5SGGmQVGUkWDnxevz0I2FJ4TeC18hRPRjKVBMb2kA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -528,6 +558,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -800,8 +832,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -976,8 +1008,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -988,8 +1020,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1222,8 +1254,8 @@ gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= -gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/1698861371_full_db_snapshot.sql similarity index 68% rename from goose/migrations/001_blobber_meta.sql rename to goose/migrations/1698861371_full_db_snapshot.sql index 470430e04..093dd857a 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -1,6 +1,5 @@ -- +goose Up -- +goose StatementBegin - SET statement_timeout = 0; SET lock_timeout = 0; SET idle_in_transaction_session_timeout = 0; @@ -11,12 +10,27 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; + + +-- +-- Name: EXTENSION pg_trgm; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION pg_trgm IS 'text similarity measurement and index searching based on trigrams'; + + +SET default_table_access_method = heap; -- -- Name: allocation_changes; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.allocation_changes ( +CREATE TABLE allocation_changes ( id bigint NOT NULL, size bigint DEFAULT 0 NOT NULL, operation character varying(20) NOT NULL, @@ -27,13 +41,13 @@ CREATE TABLE public.allocation_changes ( ); -ALTER TABLE public.allocation_changes OWNER TO blobber_user; +ALTER TABLE allocation_changes OWNER TO blobber_user; -- -- Name: allocation_changes_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.allocation_changes_id_seq +CREATE SEQUENCE allocation_changes_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -41,20 +55,20 @@ CREATE SEQUENCE public.allocation_changes_id_seq CACHE 1; -ALTER TABLE public.allocation_changes_id_seq OWNER TO blobber_user; +ALTER TABLE allocation_changes_id_seq OWNER TO blobber_user; -- -- Name: allocation_changes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.allocation_changes_id_seq OWNED BY public.allocation_changes.id; +ALTER SEQUENCE allocation_changes_id_seq OWNED BY allocation_changes.id; -- -- Name: allocation_connections; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.allocation_connections ( +CREATE TABLE allocation_connections ( id text NOT NULL, allocation_id character varying(64) NOT NULL, client_id character varying(64) NOT NULL, @@ -65,13 +79,13 @@ CREATE TABLE public.allocation_connections ( ); -ALTER TABLE public.allocation_connections OWNER TO blobber_user; +ALTER TABLE allocation_connections OWNER TO blobber_user; -- -- Name: allocations; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.allocations ( +CREATE TABLE allocations ( id character varying(64) NOT NULL, tx character varying(64) NOT NULL, size bigint DEFAULT 0 NOT NULL, @@ -94,13 +108,13 @@ CREATE TABLE public.allocations ( ); -ALTER TABLE public.allocations OWNER TO blobber_user; +ALTER TABLE allocations OWNER TO blobber_user; -- -- Name: challenge_timing; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.challenge_timing ( +CREATE TABLE challenge_timing ( challenge_id character varying(64) NOT NULL, created_at_chain bigint, created_at_blobber bigint, @@ -116,13 +130,13 @@ CREATE TABLE public.challenge_timing ( ); -ALTER TABLE public.challenge_timing OWNER TO blobber_user; +ALTER TABLE challenge_timing OWNER TO blobber_user; -- -- Name: challenges; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.challenges ( +CREATE TABLE challenges ( challenge_id character varying(64) NOT NULL, prev_challenge_id character varying(64), seed bigint DEFAULT 0 NOT NULL, @@ -147,13 +161,13 @@ CREATE TABLE public.challenges ( ); -ALTER TABLE public.challenges OWNER TO blobber_user; +ALTER TABLE challenges OWNER TO blobber_user; -- -- Name: challenges_sequence_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.challenges_sequence_seq +CREATE SEQUENCE challenges_sequence_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -161,20 +175,35 @@ CREATE SEQUENCE public.challenges_sequence_seq CACHE 1; -ALTER TABLE public.challenges_sequence_seq OWNER TO blobber_user; +ALTER TABLE challenges_sequence_seq OWNER TO blobber_user; -- -- Name: challenges_sequence_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.challenges_sequence_seq OWNED BY public.challenges.sequence; +ALTER SEQUENCE challenges_sequence_seq OWNED BY challenges.sequence; + + +-- +-- Name: client_stats; Type: TABLE; Schema: public; Owner: blobber_user +-- + +CREATE TABLE client_stats ( + client_id character varying(64) NOT NULL, + created_at bigint DEFAULT 0 NOT NULL, + total_upload bigint DEFAULT 0 NOT NULL, + total_download bigint DEFAULT 0 NOT NULL, + total_write_marker bigint DEFAULT 0 NOT NULL +); +ALTER TABLE client_stats OWNER TO blobber_user; + -- -- Name: file_stats; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.file_stats ( +CREATE TABLE file_stats ( id bigint NOT NULL, ref_id bigint, num_of_updates bigint, @@ -188,13 +217,13 @@ CREATE TABLE public.file_stats ( ); -ALTER TABLE public.file_stats OWNER TO blobber_user; +ALTER TABLE file_stats OWNER TO blobber_user; -- -- Name: file_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.file_stats_id_seq +CREATE SEQUENCE file_stats_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -202,20 +231,19 @@ CREATE SEQUENCE public.file_stats_id_seq CACHE 1; -ALTER TABLE public.file_stats_id_seq OWNER TO blobber_user; +ALTER TABLE file_stats_id_seq OWNER TO blobber_user; -- -- Name: file_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.file_stats_id_seq OWNED BY public.file_stats.id; - +ALTER SEQUENCE file_stats_id_seq OWNED BY file_stats.id; -- -- Name: marketplace_share_info; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.marketplace_share_info ( +CREATE TABLE marketplace_share_info ( id bigint NOT NULL, owner_id character varying(64) NOT NULL, client_id character varying(64) NOT NULL, @@ -228,13 +256,13 @@ CREATE TABLE public.marketplace_share_info ( ); -ALTER TABLE public.marketplace_share_info OWNER TO blobber_user; +ALTER TABLE marketplace_share_info OWNER TO blobber_user; -- -- Name: marketplace_share_info_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.marketplace_share_info_id_seq +CREATE SEQUENCE marketplace_share_info_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -242,32 +270,32 @@ CREATE SEQUENCE public.marketplace_share_info_id_seq CACHE 1; -ALTER TABLE public.marketplace_share_info_id_seq OWNER TO blobber_user; +ALTER TABLE marketplace_share_info_id_seq OWNER TO blobber_user; -- -- Name: marketplace_share_info_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.marketplace_share_info_id_seq OWNED BY public.marketplace_share_info.id; +ALTER SEQUENCE marketplace_share_info_id_seq OWNED BY marketplace_share_info.id; -- -- Name: pendings; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.pendings ( +CREATE TABLE pendings ( id text NOT NULL, pending_write bigint DEFAULT 0 NOT NULL ); -ALTER TABLE public.pendings OWNER TO blobber_user; +ALTER TABLE pendings OWNER TO blobber_user; -- -- Name: read_markers; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.read_markers ( +CREATE TABLE read_markers ( client_id character varying(64) NOT NULL, allocation_id character varying(64) NOT NULL, client_public_key character varying(128), @@ -282,25 +310,25 @@ CREATE TABLE public.read_markers ( ); -ALTER TABLE public.read_markers OWNER TO blobber_user; +ALTER TABLE read_markers OWNER TO blobber_user; -- -- Name: read_pools; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.read_pools ( +CREATE TABLE read_pools ( client_id character varying(64) NOT NULL, balance bigint NOT NULL ); -ALTER TABLE public.read_pools OWNER TO blobber_user; +ALTER TABLE read_pools OWNER TO blobber_user; -- -- Name: reference_objects; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.reference_objects ( +CREATE TABLE reference_objects ( id bigint NOT NULL, file_id text, type character varying(1), @@ -324,7 +352,7 @@ CREATE TABLE public.reference_objects ( actual_file_size bigint DEFAULT 0 NOT NULL, actual_file_hash_signature character varying(64), actual_file_hash character varying(64) NOT NULL, - mimetype character varying(64) NOT NULL, + mimetype character varying(255) NOT NULL, allocation_root character varying(64) NOT NULL, thumbnail_size bigint DEFAULT 0 NOT NULL, thumbnail_hash character varying(64) NOT NULL, @@ -337,17 +365,19 @@ CREATE TABLE public.reference_objects ( updated_at bigint, deleted_at timestamp with time zone, is_precommit boolean DEFAULT false NOT NULL, - chunk_size bigint DEFAULT 65536 NOT NULL + chunk_size bigint DEFAULT 65536 NOT NULL, + num_of_updates bigint, + num_of_block_downloads bigint ); -ALTER TABLE public.reference_objects OWNER TO blobber_user; +ALTER TABLE reference_objects OWNER TO blobber_user; -- -- Name: reference_objects_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.reference_objects_id_seq +CREATE SEQUENCE reference_objects_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -355,23 +385,22 @@ CREATE SEQUENCE public.reference_objects_id_seq CACHE 1; -ALTER TABLE public.reference_objects_id_seq OWNER TO blobber_user; +ALTER TABLE reference_objects_id_seq OWNER TO blobber_user; -- -- Name: reference_objects_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.reference_objects_id_seq OWNED BY public.reference_objects.id; +ALTER SEQUENCE reference_objects_id_seq OWNED BY reference_objects.id; -- -- Name: settings; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.settings ( +CREATE TABLE settings ( id character varying(10) NOT NULL, capacity bigint DEFAULT 0 NOT NULL, - min_lock_demand numeric DEFAULT 0.000000 NOT NULL, num_delegates bigint DEFAULT 100 NOT NULL, read_price numeric DEFAULT 0.000000 NOT NULL, write_price numeric DEFAULT 0.000000 NOT NULL, @@ -380,13 +409,13 @@ CREATE TABLE public.settings ( ); -ALTER TABLE public.settings OWNER TO blobber_user; +ALTER TABLE settings OWNER TO blobber_user; -- -- Name: terms; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.terms ( +CREATE TABLE terms ( id bigint NOT NULL, blobber_id character varying(64) NOT NULL, allocation_id character varying(64) NOT NULL, @@ -395,13 +424,13 @@ CREATE TABLE public.terms ( ); -ALTER TABLE public.terms OWNER TO blobber_user; +ALTER TABLE terms OWNER TO blobber_user; -- -- Name: terms_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.terms_id_seq +CREATE SEQUENCE terms_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -409,45 +438,64 @@ CREATE SEQUENCE public.terms_id_seq CACHE 1; -ALTER TABLE public.terms_id_seq OWNER TO blobber_user; +ALTER TABLE terms_id_seq OWNER TO blobber_user; -- -- Name: terms_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.terms_id_seq OWNED BY public.terms.id; +ALTER SEQUENCE terms_id_seq OWNED BY terms.id; -- --- Name: write_locks; Type: TABLE; Schema: public; Owner: blobber_user +-- Name: write_markers; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.write_locks ( - allocation_id character varying(64) NOT NULL, +CREATE TABLE write_markers ( + allocation_root character varying(64) NOT NULL, + prev_allocation_root character varying(64), + file_meta_root character varying(64), + allocation_id character varying(64), + size bigint, + blobber_id character varying(64), + "timestamp" bigint NOT NULL, + client_id character varying(64), + signature character varying(64), + status bigint DEFAULT 0 NOT NULL, + latest boolean DEFAULT true NOT NULL, + status_message text, + redeem_retries bigint DEFAULT 0 NOT NULL, + close_txn_id character varying(64), connection_id character varying(64), - created_at timestamp with time zone + client_key character varying(256), + sequence bigint, + created_at timestamp with time zone, + updated_at timestamp with time zone ); -ALTER TABLE public.write_locks OWNER TO blobber_user; +ALTER TABLE write_markers OWNER TO blobber_user; + +SET default_tablespace = hdd_tablespace; -- --- Name: write_markers; Type: TABLE; Schema: public; Owner: blobber_user +-- Name: write_markers_archive; Type: TABLE; Schema: public; Owner: blobber_user; Tablespace: hdd_tablespace -- -CREATE TABLE public.write_markers ( - allocation_root character varying(64) NOT NULL, +CREATE TABLE write_markers_archive ( + allocation_root character varying(64), prev_allocation_root character varying(64), file_meta_root character varying(64), allocation_id character varying(64), size bigint, blobber_id character varying(64), - "timestamp" bigint NOT NULL, + "timestamp" bigint, client_id character varying(64), signature character varying(64), - status bigint DEFAULT 0 NOT NULL, + status bigint, + latest boolean, status_message text, - redeem_retries bigint DEFAULT 0 NOT NULL, + redeem_retries bigint, close_txn_id character varying(64), connection_id character varying(64), client_key character varying(256), @@ -457,13 +505,13 @@ CREATE TABLE public.write_markers ( ); -ALTER TABLE public.write_markers OWNER TO blobber_user; +ALTER TABLE write_markers_archive OWNER TO blobber_user; -- -- Name: write_markers_sequence_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user -- -CREATE SEQUENCE public.write_markers_sequence_seq +CREATE SEQUENCE write_markers_sequence_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -471,81 +519,82 @@ CREATE SEQUENCE public.write_markers_sequence_seq CACHE 1; -ALTER TABLE public.write_markers_sequence_seq OWNER TO blobber_user; +ALTER TABLE write_markers_sequence_seq OWNER TO blobber_user; -- -- Name: write_markers_sequence_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user -- -ALTER SEQUENCE public.write_markers_sequence_seq OWNED BY public.write_markers.sequence; +ALTER SEQUENCE write_markers_sequence_seq OWNED BY write_markers.sequence; +SET default_tablespace = ''; + -- -- Name: write_pools; Type: TABLE; Schema: public; Owner: blobber_user -- -CREATE TABLE public.write_pools ( +CREATE TABLE write_pools ( allocation_id character varying(64) NOT NULL, balance bigint NOT NULL ); -ALTER TABLE public.write_pools OWNER TO blobber_user; +ALTER TABLE write_pools OWNER TO blobber_user; -- -- Name: allocation_changes id; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocation_changes ALTER COLUMN id SET DEFAULT nextval('public.allocation_changes_id_seq'::regclass); +ALTER TABLE ONLY allocation_changes ALTER COLUMN id SET DEFAULT nextval('allocation_changes_id_seq'::regclass); -- -- Name: challenges sequence; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.challenges ALTER COLUMN sequence SET DEFAULT nextval('public.challenges_sequence_seq'::regclass); +ALTER TABLE ONLY challenges ALTER COLUMN sequence SET DEFAULT nextval('challenges_sequence_seq'::regclass); -- -- Name: file_stats id; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.file_stats ALTER COLUMN id SET DEFAULT nextval('public.file_stats_id_seq'::regclass); - +ALTER TABLE ONLY file_stats ALTER COLUMN id SET DEFAULT nextval('file_stats_id_seq'::regclass); -- -- Name: marketplace_share_info id; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('public.marketplace_share_info_id_seq'::regclass); +ALTER TABLE ONLY marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('marketplace_share_info_id_seq'::regclass); -- -- Name: reference_objects id; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.reference_objects ALTER COLUMN id SET DEFAULT nextval('public.reference_objects_id_seq'::regclass); +ALTER TABLE ONLY reference_objects ALTER COLUMN id SET DEFAULT nextval('reference_objects_id_seq'::regclass); -- -- Name: terms id; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.terms ALTER COLUMN id SET DEFAULT nextval('public.terms_id_seq'::regclass); +ALTER TABLE ONLY terms ALTER COLUMN id SET DEFAULT nextval('terms_id_seq'::regclass); -- -- Name: write_markers sequence; Type: DEFAULT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.write_markers ALTER COLUMN sequence SET DEFAULT nextval('public.write_markers_sequence_seq'::regclass); +ALTER TABLE ONLY write_markers ALTER COLUMN sequence SET DEFAULT nextval('write_markers_sequence_seq'::regclass); -- -- Name: allocation_changes allocation_changes_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocation_changes +ALTER TABLE ONLY allocation_changes ADD CONSTRAINT allocation_changes_pkey PRIMARY KEY (id); @@ -553,7 +602,7 @@ ALTER TABLE ONLY public.allocation_changes -- Name: allocation_connections allocation_connections_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocation_connections +ALTER TABLE ONLY allocation_connections ADD CONSTRAINT allocation_connections_pkey PRIMARY KEY (id); @@ -561,7 +610,7 @@ ALTER TABLE ONLY public.allocation_connections -- Name: allocations allocations_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocations +ALTER TABLE ONLY allocations ADD CONSTRAINT allocations_pkey PRIMARY KEY (id); @@ -569,7 +618,7 @@ ALTER TABLE ONLY public.allocations -- Name: allocations allocations_tx_key; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocations +ALTER TABLE ONLY allocations ADD CONSTRAINT allocations_tx_key UNIQUE (tx); @@ -577,7 +626,7 @@ ALTER TABLE ONLY public.allocations -- Name: challenge_timing challenge_timing_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.challenge_timing +ALTER TABLE ONLY challenge_timing ADD CONSTRAINT challenge_timing_pkey PRIMARY KEY (challenge_id); @@ -585,7 +634,7 @@ ALTER TABLE ONLY public.challenge_timing -- Name: challenges challenges_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.challenges +ALTER TABLE ONLY challenges ADD CONSTRAINT challenges_pkey PRIMARY KEY (challenge_id); @@ -593,15 +642,23 @@ ALTER TABLE ONLY public.challenges -- Name: challenges challenges_sequence_key; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.challenges +ALTER TABLE ONLY challenges ADD CONSTRAINT challenges_sequence_key UNIQUE (sequence); +-- +-- Name: client_stats client_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user +-- + +ALTER TABLE ONLY client_stats + ADD CONSTRAINT client_stats_pkey PRIMARY KEY (client_id, created_at); + + -- -- Name: file_stats file_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.file_stats +ALTER TABLE ONLY file_stats ADD CONSTRAINT file_stats_pkey PRIMARY KEY (id); @@ -609,15 +666,14 @@ ALTER TABLE ONLY public.file_stats -- Name: file_stats file_stats_ref_id_key; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.file_stats +ALTER TABLE ONLY file_stats ADD CONSTRAINT file_stats_ref_id_key UNIQUE (ref_id); - -- -- Name: marketplace_share_info marketplace_share_info_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.marketplace_share_info +ALTER TABLE ONLY marketplace_share_info ADD CONSTRAINT marketplace_share_info_pkey PRIMARY KEY (id); @@ -625,7 +681,7 @@ ALTER TABLE ONLY public.marketplace_share_info -- Name: pendings pendings_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.pendings +ALTER TABLE ONLY pendings ADD CONSTRAINT pendings_pkey PRIMARY KEY (id); @@ -633,7 +689,7 @@ ALTER TABLE ONLY public.pendings -- Name: read_markers read_markers_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.read_markers +ALTER TABLE ONLY read_markers ADD CONSTRAINT read_markers_pkey PRIMARY KEY (client_id, allocation_id); @@ -641,7 +697,7 @@ ALTER TABLE ONLY public.read_markers -- Name: read_pools read_pools_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.read_pools +ALTER TABLE ONLY read_pools ADD CONSTRAINT read_pools_pkey PRIMARY KEY (client_id); @@ -649,7 +705,7 @@ ALTER TABLE ONLY public.read_pools -- Name: reference_objects reference_objects_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.reference_objects +ALTER TABLE ONLY reference_objects ADD CONSTRAINT reference_objects_pkey PRIMARY KEY (id); @@ -657,7 +713,7 @@ ALTER TABLE ONLY public.reference_objects -- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.settings +ALTER TABLE ONLY settings ADD CONSTRAINT settings_pkey PRIMARY KEY (id); @@ -665,23 +721,14 @@ ALTER TABLE ONLY public.settings -- Name: terms terms_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.terms +ALTER TABLE ONLY terms ADD CONSTRAINT terms_pkey PRIMARY KEY (id); - --- --- Name: write_locks write_locks_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user --- - -ALTER TABLE ONLY public.write_locks - ADD CONSTRAINT write_locks_pkey PRIMARY KEY (allocation_id); - - -- -- Name: write_markers write_markers_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.write_markers +ALTER TABLE ONLY write_markers ADD CONSTRAINT write_markers_pkey PRIMARY KEY (allocation_root, "timestamp"); @@ -689,7 +736,7 @@ ALTER TABLE ONLY public.write_markers -- Name: write_markers write_markers_sequence_key; Type: CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.write_markers +ALTER TABLE ONLY write_markers ADD CONSTRAINT write_markers_sequence_key UNIQUE (sequence); @@ -697,125 +744,132 @@ ALTER TABLE ONLY public.write_markers -- Name: idx_closed_at; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_closed_at ON public.challenge_timing USING btree (closed_at DESC); +CREATE INDEX idx_closed_at ON challenge_timing USING btree (closed_at DESC); -- -- Name: idx_created_at; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_created_at ON public.reference_objects USING btree (created_at DESC); +CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON public.reference_objects USING btree (lookup_hash); +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash); + -- --- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user +-- Name: idx_marketplace_share_info_for_client; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_path_alloc ON public.reference_objects USING btree (allocation_id, parent_path); +CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info USING btree (client_id, file_path_hash); -- --- Name: idx_marketplace_share_info_for_client; Type: INDEX; Schema: public; Owner: blobber_user +-- Name: idx_marketplace_share_info_for_owner; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_marketplace_share_info_for_client ON public.marketplace_share_info USING btree (client_id, file_path_hash); +CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info USING btree (owner_id, file_path_hash); -- --- Name: idx_marketplace_share_info_for_owner; Type: INDEX; Schema: public; Owner: blobber_user +-- Name: idx_name_gin; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_marketplace_share_info_for_owner ON public.marketplace_share_info USING btree (owner_id, file_path_hash); +CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english'::regconfig, (name)::text)); -- --- Name: idx_name_gin:gin; Type: INDEX; Schema: public; Owner: blobber_user +-- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX "idx_name_gin:gin" ON public.reference_objects USING btree (name); +CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path); -- -- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_path_alloc ON public.reference_objects USING btree (allocation_id, path); +CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path); + + +-- +-- Name: idx_path_gin_trgm; Type: INDEX; Schema: public; Owner: blobber_user +-- + +CREATE INDEX idx_path_gin_trgm ON reference_objects USING gin (path gin_trgm_ops); -- -- Name: idx_seq; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE UNIQUE INDEX idx_seq ON public.write_markers USING btree (allocation_id, sequence); +CREATE UNIQUE INDEX idx_seq ON write_markers USING btree (allocation_id, sequence); -- -- Name: idx_status; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_status ON public.challenges USING btree (status); +CREATE INDEX idx_status ON challenges USING btree (status); -- -- Name: idx_unique_allocations_tx; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE UNIQUE INDEX idx_unique_allocations_tx ON public.allocations USING btree (tx); +CREATE UNIQUE INDEX idx_unique_allocations_tx ON allocations USING btree (tx); -- -- Name: idx_updated_at; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_updated_at ON public.reference_objects USING btree (updated_at DESC); +CREATE INDEX idx_updated_at ON reference_objects USING btree (updated_at DESC); -- -- Name: idx_write_pools_cab; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_write_pools_cab ON public.write_pools USING btree (allocation_id); +CREATE INDEX idx_write_pools_cab ON write_pools USING btree (allocation_id); -- -- Name: path_idx; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX path_idx ON public.reference_objects USING btree (path); +CREATE INDEX path_idx ON reference_objects USING btree (path); -- -- Name: allocation_changes fk_allocation_connections_changes; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.allocation_changes - ADD CONSTRAINT fk_allocation_connections_changes FOREIGN KEY (connection_id) REFERENCES public.allocation_connections(id) ON DELETE CASCADE; +ALTER TABLE ONLY allocation_changes + ADD CONSTRAINT fk_allocation_connections_changes FOREIGN KEY (connection_id) REFERENCES allocation_connections(id) ON DELETE CASCADE; -- -- Name: file_stats fk_file_stats_ref; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.file_stats - ADD CONSTRAINT fk_file_stats_ref FOREIGN KEY (ref_id) REFERENCES public.reference_objects(id) ON DELETE CASCADE; +ALTER TABLE ONLY file_stats + ADD CONSTRAINT fk_file_stats_ref FOREIGN KEY (ref_id) REFERENCES reference_objects(id) ON DELETE CASCADE; -- -- Name: terms fk_terms_allocation; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- -ALTER TABLE ONLY public.terms - ADD CONSTRAINT fk_terms_allocation FOREIGN KEY (allocation_id) REFERENCES public.allocations(id); - +ALTER TABLE ONLY terms + ADD CONSTRAINT fk_terms_allocation FOREIGN KEY (allocation_id) REFERENCES allocations(id); +-- +goose StatementEnd --- --- PostgreSQL database dump complete --- +-- +goose Down +-- +goose StatementBegin -- +goose StatementEnd