From 47a8d165624227c44fea4ec90613d3e215f9b4fb Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Sat, 22 Jun 2024 01:48:22 +0530 Subject: [PATCH 01/10] adding comparision for blobbers table --- internal/api/model/zbox.go | 134 +++++++++++++++++++ internal/api/util/client/zbox_client.go | 100 ++++++++++++++ tests/api_tests/compare_0box_sharder_test.go | 48 +++++++ 3 files changed, 282 insertions(+) create mode 100644 tests/api_tests/compare_0box_sharder_test.go diff --git a/internal/api/model/zbox.go b/internal/api/model/zbox.go index ffdbd2aa8a..0b05ba50d7 100644 --- a/internal/api/model/zbox.go +++ b/internal/api/model/zbox.go @@ -1,6 +1,9 @@ package model import ( + "time" + + "github.com/0chain/common/core/currency" "github.com/0chain/gosdk/core/common" "github.com/0chain/system_test/internal/api/util/test" resty "github.com/go-resty/resty/v2" @@ -225,5 +228,136 @@ type ReferralRankOfUser struct { ReferrerID int64 `json:"referrer_id"` } +type ProviderRewards struct { + ID uint `json:"id" gorm:"primarykey"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ProviderID string `json:"provider_id" gorm:"uniqueIndex"` + Rewards currency.Coin `json:"rewards"` + TotalRewards currency.Coin `json:"total_rewards"` + RoundServiceChargeLastUpdated int64 `json:"round_service_charge_last_updated"` +} + type ZboxGraphEndpoint func(*test.SystemTest, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) type ZboxGraphBlobberEndpoint func(*test.SystemTest, string, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) + +type ZboxBlobber struct { + ID string `json:"id" validate:"hexadecimal,len=64"` + BaseURL string `json:"url"` + Terms Terms `json:"terms"` + Capacity int64 `json:"capacity"` + Allocated int64 `json:"allocated"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + PublicKey string `json:"-"` + SavedData int64 `json:"saved_data"` + DataReadLastRewardRound float64 `json:"data_read_last_reward_round"` + LastRewardDataReadRound int64 `json:"last_reward_data_read_round"` + StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` + NotAvailable bool `json:"not_available"` + ChallengesPassed int64 `json:"challenges_passed"` + ChallengesCompleted int64 `json:"challenges_completed"` + TotalStake currency.Coin `json:"total_stake"` + CreationRound int64 `json:"creation_round"` + ReadData int64 `json:"read_data"` + UsedAllocation int64 `json:"used_allocation"` + TotalOffers currency.Coin `json:"total_offers"` + StakedCapacity int64 `json:"staked_capacity"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` + CreatedAt time.Time `json:"created_at"` + IsRestricted bool `json:"is_restricted"` +} +type ZboxProviderBase struct { + ID string `gorm:"primaryKey" json:"id"` + CreatedAt time.Time + UpdatedAt time.Time + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake currency.Coin `json:"total_stake"` + Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + City *string `json:"city"` + BrandID int64 `json:"brand_id"` + GeoLocation *string `json:"geo_location"` + UserName *string `json:"user_name"` + CustomIcon *string `json:"custom_icon"` + Description *string `json:"description"` +} +type ZboxMiner struct { + ZboxProviderBase + + N2NHost string `gorm:"column:n2n_host"` + Host string + Port int + Path string + PublicKey string + ShortName string + BuildTag string + Delete bool + Fees currency.Coin + Active bool + BlocksFinalised int64 + CreationRound int64 `json:"creation_round" gorm:"index:idx_miner_creation_round"` +} + +type ZboxSharder struct { + ZboxProviderBase + + N2NHost string `gorm:"column:n2n_host"` + Host string + Port int + Path string + PublicKey string + ShortName string + BuildTag string + Delete bool + Fees currency.Coin + Active bool + CreationRound int64 `json:"creation_round" gorm:"index:idx_sharder_creation_round"` +} + +type ZboxValidator struct { + ValidatorID string `json:"validator_id"` + BaseUrl string `json:"url"` + StakeTotal currency.Coin `json:"stake_total"` + PublicKey string `json:"public_key"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` +} +type ZboxAuthorizer struct { + AuthorizerID string `json:"id"` + URL string `json:"url"` + Fee currency.Coin `json:"fee"` + LastHealthCheck int64 `json:"last_health_check"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` +} + +type ZboxSharderList struct { + Sharders []ZboxSharder `json:"sharders"` +} + +type ZboxMinerList struct { + Miners []ZboxMiner `json:"miners"` +} + +type ZboxValidatorList struct { + Validators []ZboxValidator `json:"validators"` +} + +type ZboxAuthorizerList struct { + Authorizers []ZboxAuthorizer `json:"authorizers"` +} diff --git a/internal/api/util/client/zbox_client.go b/internal/api/util/client/zbox_client.go index 496e84ca63..6a75105b8a 100644 --- a/internal/api/util/client/zbox_client.go +++ b/internal/api/util/client/zbox_client.go @@ -1284,3 +1284,103 @@ func (c *ZboxClient) GetGraphBlobberTotalRewards(t *test.SystemTest, blobberId s return &data, resp, err } + +func (c *ZboxClient) GetBlobbers(t *test.SystemTest, blobberIds string) ([]model.ZboxBlobber, *resty.Response, error) { + t.Logf("Getting blobbers using 0box...") + var data []model.ZboxBlobber + + urlBuilder := NewURLBuilder() + err := urlBuilder.MustShiftParse(c.zboxEntrypoint) + require.NoError(t, err, "URL parse error") + + urlBuilder.SetPath("/v2/blobbers") + urlBuilder.queries.Set("blobber_ids", blobberIds) + + resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ + Dst: data, + RequiredStatusCode: 200, + }, HttpGETMethod) + + return data, resp, err + +} + +func (c *ZboxClient) GetMiners(t *test.SystemTest, minerIds string) (*model.ZboxMinerList, *resty.Response, error) { + t.Logf("Getting miners using 0box...") + var data model.ZboxMinerList + + urlBuilder := NewURLBuilder() + err := urlBuilder.MustShiftParse(c.zboxEntrypoint) + require.NoError(t, err, "URL parse error") + + urlBuilder.SetPath("/v2/miners") + urlBuilder.queries.Set("miner_ids", minerIds) + + resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ + Dst: &data, + RequiredStatusCode: 200, + }, HttpGETMethod) + + return &data, resp, err + +} + +func (c *ZboxClient) GetSharders(t *test.SystemTest, sharderIds string) (*model.ZboxSharderList, *resty.Response, error) { + t.Logf("Getting sharders using 0box...") + var data model.ZboxSharderList + + urlBuilder := NewURLBuilder() + err := urlBuilder.MustShiftParse(c.zboxEntrypoint) + require.NoError(t, err, "URL parse error") + + urlBuilder.SetPath("/v2/sharders") + urlBuilder.queries.Set("sharder_ids", sharderIds) + + resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ + Dst: &data, + RequiredStatusCode: 200, + }, HttpGETMethod) + + return &data, resp, err + +} + +func (c *ZboxClient) GetAuthorizers(t *test.SystemTest, authorizerIds string) (*model.ZboxAuthorizerList, *resty.Response, error) { + t.Logf("Getting authorizers using 0box...") + var data model.ZboxAuthorizerList + + urlBuilder := NewURLBuilder() + err := urlBuilder.MustShiftParse(c.zboxEntrypoint) + require.NoError(t, err, "URL parse error") + + urlBuilder.SetPath("/v2/authorizers") + urlBuilder.queries.Set("authorizer_ids", authorizerIds) + + resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ + Dst: &data, + RequiredStatusCode: 200, + }, HttpGETMethod) + + return &data, resp, err + +} + +func (c *ZboxClient) GetValidators(t *test.SystemTest, validatorIds string) (*model.ZboxValidatorList, *resty.Response, error) { + t.Logf("Getting validators using 0box...") + var data model.ZboxValidatorList + + urlBuilder := NewURLBuilder() + err := urlBuilder.MustShiftParse(c.zboxEntrypoint) + require.NoError(t, err, "URL parse error") + + urlBuilder.SetPath("/v2/validators") + urlBuilder.queries.Set("validator_ids", validatorIds) + + resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ + Dst: &data, + RequiredStatusCode: 200, + }, HttpGETMethod) + + return &data, resp, err + +} diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go new file mode 100644 index 0000000000..3ab495f7cd --- /dev/null +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -0,0 +1,48 @@ +package api_tests + +import ( + "encoding/json" + "testing" + + "github.com/0chain/system_test/internal/api/util/client" + "github.com/0chain/system_test/internal/api/util/test" + "github.com/stretchr/testify/require" +) + +func Compares0boxTablesWithSharder(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + t.SetSmokeTests("Compare 0box tables with sharder tables") + + t.RunSequentially("Compare blobbers tables", func(t *test.SystemTest) { + blobbersTable_Sharder, resp, err := apiClient.V1SCRestGetAllBlobbers(t, client.HttpOkStatus) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode()) + var blobberIds []string + for _, blobber := range blobbersTable_Sharder { + blobberIds = append(blobberIds, blobber.ID) + } + blobberIdsJson, err := json.Marshal(blobberIds) + require.NoError(t, err) + blobberIdsStr := string(blobberIdsJson) + blobbersTable_0box, resp, err := zboxClient.GetBlobbers(t, blobberIdsStr) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode()) + require.Equal(t, len(blobbersTable_Sharder), len(blobbersTable_0box)) + for i, blobber := range blobbersTable_Sharder { + require.Equal(t, blobber.ID, blobbersTable_0box[i].ID) + require.Equal(t, blobber.BaseURL, blobbersTable_0box[i].BaseURL) + require.Equal(t, blobber.Terms, blobbersTable_0box[i].Terms) + require.Equal(t, blobber.Capacity, blobbersTable_0box[i].Capacity) + require.Equal(t, blobber.Allocated, blobbersTable_0box[i].Allocated) + require.Equal(t, blobber.LastHealthCheck, blobbersTable_0box[i].LastHealthCheck) + require.Equal(t, blobber.PublicKey, blobbersTable_0box[i].PublicKey) + require.Equal(t, blobber.StakePoolSettings, blobbersTable_0box[i].StakePoolSettings) + require.Equal(t, blobber.TotalStake, blobbersTable_0box[i].TotalStake) + require.Equal(t, blobber.SavedData, blobbersTable_0box[i].SavedData) + require.Equal(t, blobber.ReadData, blobbersTable_0box[i].ReadData) + require.Equal(t, blobber.ChallengesPassed, blobbersTable_0box[i].ChallengesPassed) + require.Equal(t, blobber.ChallengesCompleted, blobbersTable_0box[i].ChallengesCompleted) + } + + }) +} From 4fb56b4d7af4938c635c59d5733b8894307651f4 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Tue, 25 Jun 2024 21:00:19 +0530 Subject: [PATCH 02/10] adding changes --- internal/api/model/compare.go | 123 ++++++++++++++++++ internal/api/model/zbox.go | 121 ------------------ internal/api/util/client/api_client.go | 50 ++++++++ internal/api/util/client/zbox_client.go | 124 ++++++------------- tests/api_tests/compare_0box_sharder_test.go | 33 +---- 5 files changed, 215 insertions(+), 236 deletions(-) create mode 100644 internal/api/model/compare.go diff --git a/internal/api/model/compare.go b/internal/api/model/compare.go new file mode 100644 index 0000000000..c313b51e50 --- /dev/null +++ b/internal/api/model/compare.go @@ -0,0 +1,123 @@ +package model + +import ( + "time" + + "github.com/0chain/common/core/currency" + "github.com/0chain/gosdk/core/common" +) + +type Blobber struct { + ID string `json:"id" validate:"hexadecimal,len=64"` + BaseURL string `json:"url"` + Terms Terms `json:"terms"` + Capacity int64 `json:"capacity"` + Allocated int64 `json:"allocated"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + PublicKey string `json:"-"` + SavedData int64 `json:"saved_data"` + DataReadLastRewardRound float64 `json:"data_read_last_reward_round"` + LastRewardDataReadRound int64 `json:"last_reward_data_read_round"` + StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` + NotAvailable bool `json:"not_available"` + ChallengesPassed int64 `json:"challenges_passed"` + ChallengesCompleted int64 `json:"challenges_completed"` + TotalStake currency.Coin `json:"total_stake"` + CreationRound int64 `json:"creation_round"` + ReadData int64 `json:"read_data"` + UsedAllocation int64 `json:"used_allocation"` + TotalOffers currency.Coin `json:"total_offers"` + StakedCapacity int64 `json:"staked_capacity"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` + CreatedAt time.Time `json:"created_at"` + IsRestricted bool `json:"is_restricted"` +} +type ProviderBase struct { + ID string `gorm:"primaryKey" json:"id"` + CreatedAt time.Time + UpdatedAt time.Time + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake currency.Coin `json:"total_stake"` + Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` +} +type Miner struct { + ProviderBase + + N2NHost string `gorm:"column:n2n_host"` + Host string + Port int + Path string + PublicKey string + ShortName string + BuildTag string + Delete bool + Fees currency.Coin + Active bool + BlocksFinalised int64 + CreationRound int64 `json:"creation_round" gorm:"index:idx_miner_creation_round"` +} + +type Sharder struct { + ProviderBase + + N2NHost string `gorm:"column:n2n_host"` + Host string + Port int + Path string + PublicKey string + ShortName string + BuildTag string + Delete bool + Fees currency.Coin + Active bool + CreationRound int64 `json:"creation_round" gorm:"index:idx_sharder_creation_round"` +} + +type Validator struct { + ValidatorID string `json:"validator_id"` + BaseUrl string `json:"url"` + StakeTotal currency.Coin `json:"stake_total"` + PublicKey string `json:"public_key"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` +} +type Authorizer struct { + AuthorizerID string `json:"id"` + URL string `json:"url"` + Fee currency.Coin `json:"fee"` + LastHealthCheck int64 `json:"last_health_check"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` +} + +type SharderList struct { + Sharders []Sharder `json:"sharders"` +} + +type MinerList struct { + Miners []Miner `json:"miners"` +} + +type ValidatorList struct { + Validators []Validator `json:"validators"` +} + +type AuthorizerList struct { + Authorizers []Authorizer `json:"authorizers"` +} diff --git a/internal/api/model/zbox.go b/internal/api/model/zbox.go index 0b05ba50d7..46c1340ec6 100644 --- a/internal/api/model/zbox.go +++ b/internal/api/model/zbox.go @@ -240,124 +240,3 @@ type ProviderRewards struct { type ZboxGraphEndpoint func(*test.SystemTest, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) type ZboxGraphBlobberEndpoint func(*test.SystemTest, string, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) - -type ZboxBlobber struct { - ID string `json:"id" validate:"hexadecimal,len=64"` - BaseURL string `json:"url"` - Terms Terms `json:"terms"` - Capacity int64 `json:"capacity"` - Allocated int64 `json:"allocated"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - PublicKey string `json:"-"` - SavedData int64 `json:"saved_data"` - DataReadLastRewardRound float64 `json:"data_read_last_reward_round"` - LastRewardDataReadRound int64 `json:"last_reward_data_read_round"` - StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` - NotAvailable bool `json:"not_available"` - ChallengesPassed int64 `json:"challenges_passed"` - ChallengesCompleted int64 `json:"challenges_completed"` - TotalStake currency.Coin `json:"total_stake"` - CreationRound int64 `json:"creation_round"` - ReadData int64 `json:"read_data"` - UsedAllocation int64 `json:"used_allocation"` - TotalOffers currency.Coin `json:"total_offers"` - StakedCapacity int64 `json:"staked_capacity"` - TotalServiceCharge currency.Coin `json:"total_service_charge"` - UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` - CreatedAt time.Time `json:"created_at"` - IsRestricted bool `json:"is_restricted"` -} -type ZboxProviderBase struct { - ID string `gorm:"primaryKey" json:"id"` - CreatedAt time.Time - UpdatedAt time.Time - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` - TotalStake currency.Coin `json:"total_stake"` - Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` - Downtime uint64 `json:"downtime"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - City *string `json:"city"` - BrandID int64 `json:"brand_id"` - GeoLocation *string `json:"geo_location"` - UserName *string `json:"user_name"` - CustomIcon *string `json:"custom_icon"` - Description *string `json:"description"` -} -type ZboxMiner struct { - ZboxProviderBase - - N2NHost string `gorm:"column:n2n_host"` - Host string - Port int - Path string - PublicKey string - ShortName string - BuildTag string - Delete bool - Fees currency.Coin - Active bool - BlocksFinalised int64 - CreationRound int64 `json:"creation_round" gorm:"index:idx_miner_creation_round"` -} - -type ZboxSharder struct { - ZboxProviderBase - - N2NHost string `gorm:"column:n2n_host"` - Host string - Port int - Path string - PublicKey string - ShortName string - BuildTag string - Delete bool - Fees currency.Coin - Active bool - CreationRound int64 `json:"creation_round" gorm:"index:idx_sharder_creation_round"` -} - -type ZboxValidator struct { - ValidatorID string `json:"validator_id"` - BaseUrl string `json:"url"` - StakeTotal currency.Coin `json:"stake_total"` - PublicKey string `json:"public_key"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` - TotalServiceCharge currency.Coin `json:"total_service_charge"` - UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` -} -type ZboxAuthorizer struct { - AuthorizerID string `json:"id"` - URL string `json:"url"` - Fee currency.Coin `json:"fee"` - LastHealthCheck int64 `json:"last_health_check"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` -} - -type ZboxSharderList struct { - Sharders []ZboxSharder `json:"sharders"` -} - -type ZboxMinerList struct { - Miners []ZboxMiner `json:"miners"` -} - -type ZboxValidatorList struct { - Validators []ZboxValidator `json:"validators"` -} - -type ZboxAuthorizerList struct { - Authorizers []ZboxAuthorizer `json:"authorizers"` -} diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 3b325674f5..d460dd962d 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "net/url" + "reflect" "strconv" "strings" "time" @@ -61,6 +62,7 @@ const ( PartitionSizeFrequency = "/v1/screst/:sc_address/parition-size-frequency" BlobberPartitionSelectionFrequency = "/v1/screst/:sc_address/blobber-selection-frequency" GetAllChallenges = "/v1/screst/:sc_address/all-challenges" + GetQueryData = "/v1/screst/:sc_address/query-data" ) // Contains all used service providers @@ -2231,3 +2233,51 @@ func (c *APIClient) BurnZcn(t *test.SystemTest, wallet *model.Wallet, address st wallet.IncNonce() return burnZcnTransactionGetConfirmationResponse.Hash } + +func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) (*[]interface{}, *resty.Response, error) { + t.Logf("Querying data from Sharder...") + + extractFields := func(model interface{}) string { + val := reflect.ValueOf(model) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + if val.Kind() != reflect.Struct { + return "" + } + var fieldNames []string + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + fieldNames = append(fieldNames, typ.Field(i).Name) + } + return strings.Join(fieldNames, ", ") + } + + urlBuilder := NewURLBuilder(). + SetPath(GetQueryData). + SetPathVariable("sc_address", StorageSmartContractAddress) + + var data []interface{} + var tableEntity interface{} + switch tableName { + case "blobber": + tableEntity = model.Blobber{} + case "miner": + tableEntity = model.Miner{} + case "authorizer": + tableEntity = model.Authorizer{} + case "validator": + tableEntity = model.Validator{} + case "sharder": + tableEntity = model.Sharder{} + } + urlBuilder.queries.Set("table", tableName) + urlBuilder.queries.Set("fields", extractFields(tableEntity)) + + resp, err := c.executeForAllServiceProviders(t, urlBuilder, &model.ExecutionRequest{ + Dst: &data, + RequiredStatusCode: 200, + }, HttpGETMethod, SharderServiceProvider) + + return &data, resp, err +} diff --git a/internal/api/util/client/zbox_client.go b/internal/api/util/client/zbox_client.go index 6a75105b8a..5af89aa83e 100644 --- a/internal/api/util/client/zbox_client.go +++ b/internal/api/util/client/zbox_client.go @@ -2,7 +2,9 @@ package client import ( "fmt" + "reflect" "strconv" + "strings" "github.com/0chain/system_test/internal/api/model" "github.com/0chain/system_test/internal/api/util/test" @@ -1285,96 +1287,45 @@ func (c *ZboxClient) GetGraphBlobberTotalRewards(t *test.SystemTest, blobberId s return &data, resp, err } -func (c *ZboxClient) GetBlobbers(t *test.SystemTest, blobberIds string) ([]model.ZboxBlobber, *resty.Response, error) { - t.Logf("Getting blobbers using 0box...") - var data []model.ZboxBlobber - - urlBuilder := NewURLBuilder() - err := urlBuilder.MustShiftParse(c.zboxEntrypoint) - require.NoError(t, err, "URL parse error") - - urlBuilder.SetPath("/v2/blobbers") - urlBuilder.queries.Set("blobber_ids", blobberIds) - - resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ - Dst: data, - RequiredStatusCode: 200, - }, HttpGETMethod) - - return data, resp, err - -} - -func (c *ZboxClient) GetMiners(t *test.SystemTest, minerIds string) (*model.ZboxMinerList, *resty.Response, error) { - t.Logf("Getting miners using 0box...") - var data model.ZboxMinerList - - urlBuilder := NewURLBuilder() - err := urlBuilder.MustShiftParse(c.zboxEntrypoint) - require.NoError(t, err, "URL parse error") - - urlBuilder.SetPath("/v2/miners") - urlBuilder.queries.Set("miner_ids", minerIds) - - resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ - Dst: &data, - RequiredStatusCode: 200, - }, HttpGETMethod) - - return &data, resp, err - -} - -func (c *ZboxClient) GetSharders(t *test.SystemTest, sharderIds string) (*model.ZboxSharderList, *resty.Response, error) { - t.Logf("Getting sharders using 0box...") - var data model.ZboxSharderList - - urlBuilder := NewURLBuilder() - err := urlBuilder.MustShiftParse(c.zboxEntrypoint) - require.NoError(t, err, "URL parse error") - - urlBuilder.SetPath("/v2/sharders") - urlBuilder.queries.Set("sharder_ids", sharderIds) - - resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ - Dst: &data, - RequiredStatusCode: 200, - }, HttpGETMethod) - - return &data, resp, err - -} - -func (c *ZboxClient) GetAuthorizers(t *test.SystemTest, authorizerIds string) (*model.ZboxAuthorizerList, *resty.Response, error) { - t.Logf("Getting authorizers using 0box...") - var data model.ZboxAuthorizerList - - urlBuilder := NewURLBuilder() - err := urlBuilder.MustShiftParse(c.zboxEntrypoint) - require.NoError(t, err, "URL parse error") - - urlBuilder.SetPath("/v2/authorizers") - urlBuilder.queries.Set("authorizer_ids", authorizerIds) - - resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ - Dst: &data, - RequiredStatusCode: 200, - }, HttpGETMethod) - - return &data, resp, err - -} - -func (c *ZboxClient) GetValidators(t *test.SystemTest, validatorIds string) (*model.ZboxValidatorList, *resty.Response, error) { - t.Logf("Getting validators using 0box...") - var data model.ZboxValidatorList +func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) (*[]interface{}, *resty.Response, error) { + t.Logf("Querying data from 0box...") + + extractFields := func(model interface{}) string { + val := reflect.ValueOf(model) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + if val.Kind() != reflect.Struct { + return "" + } + var fieldNames []string + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + fieldNames = append(fieldNames, typ.Field(i).Name) + } + return strings.Join(fieldNames, ", ") + } urlBuilder := NewURLBuilder() err := urlBuilder.MustShiftParse(c.zboxEntrypoint) require.NoError(t, err, "URL parse error") - - urlBuilder.SetPath("/v2/validators") - urlBuilder.queries.Set("validator_ids", validatorIds) + urlBuilder.SetPath("/v2/queryData") + var data []interface{} + var tableEntity interface{} + switch tableName { + case "blobber": + tableEntity = model.Blobber{} + case "miner": + tableEntity = model.Miner{} + case "authorizer": + tableEntity = model.Authorizer{} + case "validator": + tableEntity = model.Validator{} + case "sharder": + tableEntity = model.Sharder{} + } + urlBuilder.queries.Set("table", tableName) + urlBuilder.queries.Set("fields", extractFields(tableEntity)) resp, err := c.executeForServiceProvider(t, urlBuilder.String(), model.ExecutionRequest{ Dst: &data, @@ -1382,5 +1333,4 @@ func (c *ZboxClient) GetValidators(t *test.SystemTest, validatorIds string) (*mo }, HttpGETMethod) return &data, resp, err - } diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 3ab495f7cd..58229ec46e 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -1,10 +1,8 @@ package api_tests import ( - "encoding/json" "testing" - "github.com/0chain/system_test/internal/api/util/client" "github.com/0chain/system_test/internal/api/util/test" "github.com/stretchr/testify/require" ) @@ -14,35 +12,14 @@ func Compares0boxTablesWithSharder(testSetup *testing.T) { t.SetSmokeTests("Compare 0box tables with sharder tables") t.RunSequentially("Compare blobbers tables", func(t *test.SystemTest) { - blobbersTable_Sharder, resp, err := apiClient.V1SCRestGetAllBlobbers(t, client.HttpOkStatus) + blobbersTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, "blobbers") require.NoError(t, err) require.Equal(t, 200, resp.StatusCode()) - var blobberIds []string - for _, blobber := range blobbersTable_Sharder { - blobberIds = append(blobberIds, blobber.ID) - } - blobberIdsJson, err := json.Marshal(blobberIds) - require.NoError(t, err) - blobberIdsStr := string(blobberIdsJson) - blobbersTable_0box, resp, err := zboxClient.GetBlobbers(t, blobberIdsStr) + blobbersTable_0box, resp, err := zboxClient.QueryDataFrom0box(t, "blobbers") require.NoError(t, err) require.Equal(t, 200, resp.StatusCode()) - require.Equal(t, len(blobbersTable_Sharder), len(blobbersTable_0box)) - for i, blobber := range blobbersTable_Sharder { - require.Equal(t, blobber.ID, blobbersTable_0box[i].ID) - require.Equal(t, blobber.BaseURL, blobbersTable_0box[i].BaseURL) - require.Equal(t, blobber.Terms, blobbersTable_0box[i].Terms) - require.Equal(t, blobber.Capacity, blobbersTable_0box[i].Capacity) - require.Equal(t, blobber.Allocated, blobbersTable_0box[i].Allocated) - require.Equal(t, blobber.LastHealthCheck, blobbersTable_0box[i].LastHealthCheck) - require.Equal(t, blobber.PublicKey, blobbersTable_0box[i].PublicKey) - require.Equal(t, blobber.StakePoolSettings, blobbersTable_0box[i].StakePoolSettings) - require.Equal(t, blobber.TotalStake, blobbersTable_0box[i].TotalStake) - require.Equal(t, blobber.SavedData, blobbersTable_0box[i].SavedData) - require.Equal(t, blobber.ReadData, blobbersTable_0box[i].ReadData) - require.Equal(t, blobber.ChallengesPassed, blobbersTable_0box[i].ChallengesPassed) - require.Equal(t, blobber.ChallengesCompleted, blobbersTable_0box[i].ChallengesCompleted) - } + require.Equal(t, blobbersTable_Sharder, blobbersTable_0box) - }) + }, + ) } From ea4c111738423569b238288dea8f4c3b56c21f89 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Mon, 8 Jul 2024 02:34:38 +0530 Subject: [PATCH 03/10] making things work --- internal/api/model/compare.go | 102 +++++++++---------- internal/api/util/client/api_client.go | 37 +++++-- internal/api/util/client/zbox_client.go | 27 ++++- tests/api_tests/compare_0box_sharder_test.go | 47 ++++++++- 4 files changed, 139 insertions(+), 74 deletions(-) diff --git a/internal/api/model/compare.go b/internal/api/model/compare.go index c313b51e50..7eeb516596 100644 --- a/internal/api/model/compare.go +++ b/internal/api/model/compare.go @@ -8,66 +8,60 @@ import ( ) type Blobber struct { - ID string `json:"id" validate:"hexadecimal,len=64"` - BaseURL string `json:"url"` - Terms Terms `json:"terms"` - Capacity int64 `json:"capacity"` - Allocated int64 `json:"allocated"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - PublicKey string `json:"-"` - SavedData int64 `json:"saved_data"` - DataReadLastRewardRound float64 `json:"data_read_last_reward_round"` - LastRewardDataReadRound int64 `json:"last_reward_data_read_round"` - StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` - NotAvailable bool `json:"not_available"` - ChallengesPassed int64 `json:"challenges_passed"` - ChallengesCompleted int64 `json:"challenges_completed"` - TotalStake currency.Coin `json:"total_stake"` - CreationRound int64 `json:"creation_round"` - ReadData int64 `json:"read_data"` - UsedAllocation int64 `json:"used_allocation"` - TotalOffers currency.Coin `json:"total_offers"` - StakedCapacity int64 `json:"staked_capacity"` - TotalServiceCharge currency.Coin `json:"total_service_charge"` - UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` - CreatedAt time.Time `json:"created_at"` - IsRestricted bool `json:"is_restricted"` -} -type ProviderBase struct { - ID string `gorm:"primaryKey" json:"id"` - CreatedAt time.Time - UpdatedAt time.Time - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` - TotalStake currency.Coin `json:"total_stake"` - Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` - Downtime uint64 `json:"downtime"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` + ID string `gorm:"primaryKey" json:"id"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates string `json:"num_delegates"` + ServiceCharge string `json:"service_charge"` + TotalStake string `json:"total_stake"` + // Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` + Downtime uint64 `json:"downtime"` + LastHealthCheck string `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + BaseURL string `json:"base_url"` + ReadPrice string `json:"read_price"` + WritePrice string `json:"write_price"` + Capacity int64 `json:"capacity"` + Allocated int64 `json:"allocated"` } type Miner struct { - ProviderBase + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` - N2NHost string `gorm:"column:n2n_host"` - Host string - Port int - Path string - PublicKey string - ShortName string - BuildTag string - Delete bool - Fees currency.Coin - Active bool - BlocksFinalised int64 - CreationRound int64 `json:"creation_round" gorm:"index:idx_miner_creation_round"` + TotalStake int64 `json:"total_stake"` + Downtime int64 `json:"downtime"` + LastHealthCheck int64 `json:"last_health_check"` + N2NHost string `json:"n2n_host"` + Host string `json:"host"` + Port int64 `json:"port"` + Path string `json:"path"` + PublicKey string `json:"public_key"` + ShortName string `json:"short_name"` + BuildTag string `json:"build_tag"` + Delete bool `json:"delete"` + Fees currency.Coin `json:"fees"` + Active bool `json:"active"` + BlocksFinalised int64 `json:"blocks_finalised"` + CreationRound int64 `json:"creation_round" gorm:"index:idx_miner_creation_round"` } type Sharder struct { - ProviderBase + ID string `gorm:"primaryKey" json:"id"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates string `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake string `json:"total_stake"` + // Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` N2NHost string `gorm:"column:n2n_host"` Host string diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index d460dd962d..4fd30ec5ec 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -2234,8 +2234,8 @@ func (c *APIClient) BurnZcn(t *test.SystemTest, wallet *model.Wallet, address st return burnZcnTransactionGetConfirmationResponse.Hash } -func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) (*[]interface{}, *resty.Response, error) { - t.Logf("Querying data from Sharder...") +func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) ([]interface{}, *resty.Response, error) { + t.Log("Querying data from Sharder...") extractFields := func(model interface{}) string { val := reflect.ValueOf(model) @@ -2248,16 +2248,22 @@ func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) ( var fieldNames []string typ := val.Type() for i := 0; i < val.NumField(); i++ { - fieldNames = append(fieldNames, typ.Field(i).Name) - } - return strings.Join(fieldNames, ", ") + field := typ.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag != "" && jsonTag != "-" { + tagParts := strings.Split(jsonTag, ",") + fieldNames = append(fieldNames, tagParts[0]) + } else { + fieldNames = append(fieldNames, field.Name) + } + } + return strings.Join(fieldNames, ",") } - urlBuilder := NewURLBuilder(). SetPath(GetQueryData). SetPathVariable("sc_address", StorageSmartContractAddress) - var data []interface{} + var data interface{} var tableEntity interface{} switch tableName { case "blobber": @@ -2271,13 +2277,24 @@ func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) ( case "sharder": tableEntity = model.Sharder{} } - urlBuilder.queries.Set("table", tableName) + urlBuilder.queries.Set("entity", tableName) urlBuilder.queries.Set("fields", extractFields(tableEntity)) - resp, err := c.executeForAllServiceProviders(t, urlBuilder, &model.ExecutionRequest{ Dst: &data, RequiredStatusCode: 200, + Headers: map[string]string{ + "Accept-Encoding": "", + }, }, HttpGETMethod, SharderServiceProvider) + var result []interface{} + switch v := data.(type) { + case []interface{}: + for _, value := range v { + result = append(result, value) + } + default: + t.Error("Invalid response from Sharder") + } - return &data, resp, err + return result, resp, err } diff --git a/internal/api/util/client/zbox_client.go b/internal/api/util/client/zbox_client.go index 5af89aa83e..0e0f111077 100644 --- a/internal/api/util/client/zbox_client.go +++ b/internal/api/util/client/zbox_client.go @@ -1287,7 +1287,7 @@ func (c *ZboxClient) GetGraphBlobberTotalRewards(t *test.SystemTest, blobberId s return &data, resp, err } -func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) (*[]interface{}, *resty.Response, error) { +func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) ([]interface{}, *resty.Response, error) { t.Logf("Querying data from 0box...") extractFields := func(model interface{}) string { @@ -1301,16 +1301,23 @@ func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) (*[ var fieldNames []string typ := val.Type() for i := 0; i < val.NumField(); i++ { - fieldNames = append(fieldNames, typ.Field(i).Name) + field := typ.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag != "" && jsonTag != "-" { + tagParts := strings.Split(jsonTag, ",") + fieldNames = append(fieldNames, tagParts[0]) + } else { + fieldNames = append(fieldNames, field.Name) + } } - return strings.Join(fieldNames, ", ") + return strings.Join(fieldNames, ",") } urlBuilder := NewURLBuilder() err := urlBuilder.MustShiftParse(c.zboxEntrypoint) require.NoError(t, err, "URL parse error") urlBuilder.SetPath("/v2/queryData") - var data []interface{} + var data interface{} var tableEntity interface{} switch tableName { case "blobber": @@ -1332,5 +1339,15 @@ func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) (*[ RequiredStatusCode: 200, }, HttpGETMethod) - return &data, resp, err + var result []interface{} + switch v := data.(type) { + case []interface{}: + for _, value := range v { + result = append(result, value) + } + default: + t.Error("Invalid response from Sharder") + } + + return result, resp, err } diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 58229ec46e..d334449da6 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -7,18 +7,55 @@ import ( "github.com/stretchr/testify/require" ) -func Compares0boxTablesWithSharder(testSetup *testing.T) { +func TestCompares0boxTablesWithSharder(testSetup *testing.T) { t := test.NewSystemTest(testSetup) t.SetSmokeTests("Compare 0box tables with sharder tables") - t.RunSequentially("Compare blobbers tables", func(t *test.SystemTest) { - blobbersTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, "blobbers") + t.RunSequentially("Compare Miner tables", func(t *test.SystemTest) { + + minersTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, "miner") require.NoError(t, err) require.Equal(t, 200, resp.StatusCode()) - blobbersTable_0box, resp, err := zboxClient.QueryDataFrom0box(t, "blobbers") + + minersTable_0box, resp, err := zboxClient.QueryDataFrom0box(t, "miner") require.NoError(t, err) require.Equal(t, 200, resp.StatusCode()) - require.Equal(t, blobbersTable_Sharder, blobbersTable_0box) + + require.Equal(t, len(minersTable_Sharder), len(minersTable_0box)) + // t.Logf("minersTable_Sharder: %+v", minersTable_Sharder) + // t.Logf("minersTable_0box: %+v", minersTable_0box) + + var miners_Sharder, miners_0box map[string]map[string]interface{} + miners_Sharder = make(map[string]map[string]interface{}) + miners_0box = make(map[string]map[string]interface{}) + for i := 0; i < len(minersTable_Sharder); i++ { + m_sharder := minersTable_Sharder[i].(map[string]interface{}) + m_0box := minersTable_0box[i].(map[string]interface{}) + // for k := range b_sharder { + // t.Logf("k: %s", k) + // if k == "ID" { + // t.Logf("IDxxx: %s", b_sharder[k].(string)) + // } + // } + // t.Logf("ID: %s", b_sharder["ID"].(string)) + miners_Sharder[m_sharder["ID"].(string)] = m_sharder + miners_0box[m_0box["id"].(string)] = m_0box + } + for k, v := range miners_Sharder { + for k1, v1 := range v { + if v1 != nil && miners_0box[k][k1] != nil { + require.Equal(t, v1, miners_0box[k][k1]) + if v1 != miners_0box[k][k1] { + t.Logf("id:%s and key %s", k, k1) + } + } + + } + + // t.Logf("blobbersTable_Sharder: %+v", blobbersTable_Sharder) + // t.Logf("blobbersTable_0box: %+v", blobbersTable_0box) + + require.Equal(t, 1, 0) }, ) From 7292cc4eb9f29f64793e7822199aea0c0cc5f6ae Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Mon, 8 Jul 2024 02:35:36 +0530 Subject: [PATCH 04/10] error fix --- tests/api_tests/compare_0box_sharder_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index d334449da6..31bd83398b 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -46,7 +46,8 @@ func TestCompares0boxTablesWithSharder(testSetup *testing.T) { if v1 != nil && miners_0box[k][k1] != nil { require.Equal(t, v1, miners_0box[k][k1]) if v1 != miners_0box[k][k1] { - t.Logf("id:%s and key %s", k, k1) + t.Logf("id:%s and key:%s", k, k1) + } } } From e5c409c445685978eac4cfadb371772f1b812799 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Mon, 8 Jul 2024 02:37:57 +0530 Subject: [PATCH 05/10] edit --- tests/api_tests/compare_0box_sharder_test.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 31bd83398b..22167c44a3 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -31,33 +31,20 @@ func TestCompares0boxTablesWithSharder(testSetup *testing.T) { for i := 0; i < len(minersTable_Sharder); i++ { m_sharder := minersTable_Sharder[i].(map[string]interface{}) m_0box := minersTable_0box[i].(map[string]interface{}) - // for k := range b_sharder { - // t.Logf("k: %s", k) - // if k == "ID" { - // t.Logf("IDxxx: %s", b_sharder[k].(string)) - // } - // } - // t.Logf("ID: %s", b_sharder["ID"].(string)) miners_Sharder[m_sharder["ID"].(string)] = m_sharder miners_0box[m_0box["id"].(string)] = m_0box } for k, v := range miners_Sharder { for k1, v1 := range v { if v1 != nil && miners_0box[k][k1] != nil { - require.Equal(t, v1, miners_0box[k][k1]) if v1 != miners_0box[k][k1] { t.Logf("id:%s and key:%s", k, k1) } + require.Equal(t, v1, miners_0box[k][k1]) } } } - - // t.Logf("blobbersTable_Sharder: %+v", blobbersTable_Sharder) - // t.Logf("blobbersTable_0box: %+v", blobbersTable_0box) - - require.Equal(t, 1, 0) - }, ) } From c42b08743676248122607ff5d112ab306587fc09 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Tue, 9 Jul 2024 01:54:40 +0530 Subject: [PATCH 06/10] adding remaining test --- internal/api/model/compare.go | 164 +++++++++++++++---- internal/api/util/client/api_client.go | 17 ++ internal/api/util/client/zbox_client.go | 17 ++ tests/api_tests/compare_0box_sharder_test.go | 138 ++++++++++++---- 4 files changed, 271 insertions(+), 65 deletions(-) diff --git a/internal/api/model/compare.go b/internal/api/model/compare.go index 7eeb516596..acd7ed117b 100644 --- a/internal/api/model/compare.go +++ b/internal/api/model/compare.go @@ -8,33 +8,50 @@ import ( ) type Blobber struct { - ID string `gorm:"primaryKey" json:"id"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates string `json:"num_delegates"` - ServiceCharge string `json:"service_charge"` - TotalStake string `json:"total_stake"` - // Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` - Downtime uint64 `json:"downtime"` - LastHealthCheck string `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - BaseURL string `json:"base_url"` - ReadPrice string `json:"read_price"` - WritePrice string `json:"write_price"` - Capacity int64 `json:"capacity"` - Allocated int64 `json:"allocated"` + ID string `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake currency.Coin `json:"total_stake"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + BaseURL string `json:"base_url" gorm:"uniqueIndex"` + ReadPrice currency.Coin `json:"read_price"` + WritePrice currency.Coin `json:"write_price"` + Capacity int64 `json:"capacity"` + Allocated int64 `json:"allocated"` + SavedData int64 `json:"saved_data"` + ReadData int64 `json:"read_data"` + NotAvailable bool `json:"not_available"` + IsRestricted bool `json:"is_restricted"` + OffersTotal currency.Coin `json:"offers_total"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + ChallengesPassed uint64 `json:"challenges_passed"` + ChallengesCompleted uint64 `json:"challenges_completed"` + OpenChallenges uint64 `json:"open_challenges"` + RankMetric float64 `json:"rank_metric"` + TotalBlockRewards currency.Coin `json:"total_block_rewards"` + TotalStorageIncome currency.Coin `json:"total_storage_income"` + TotalReadIncome currency.Coin `json:"total_read_income"` + TotalSlashedStake currency.Coin `json:"total_slashed_stake"` + CreationRound int64 `json:"creation_round"` } type Miner struct { - ID string `json:"id"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - DelegateWallet string `json:"delegate_wallet"` - + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` TotalStake int64 `json:"total_stake"` Downtime int64 `json:"downtime"` LastHealthCheck int64 `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` N2NHost string `json:"n2n_host"` Host string `json:"host"` Port int64 `json:"port"` @@ -99,19 +116,104 @@ type Authorizer struct { NumDelegates int `json:"num_delegates"` ServiceCharge float64 `json:"service_charge"` } - -type SharderList struct { - Sharders []Sharder `json:"sharders"` +type User struct { + ID uint `json:"id" gorm:"primarykey"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + UserID string `json:"user_id" gorm:"uniqueIndex"` + TxnHash string `json:"txn_hash"` + Balance currency.Coin `json:"balance"` + Round int64 `json:"round"` + Nonce int64 `json:"nonce"` + MintNonce int64 `json:"mint_nonce"` +} +type UserSnapshot struct { + UserID string `json:"user_id" gorm:"uniqueIndex"` + Round int64 `json:"round"` + TotalReward int64 `json:"total_reward"` + CollectedReward int64 `json:"collected_reward"` + TotalStake int64 `json:"total_stake"` + ReadPoolTotal int64 `json:"read_pool_total"` + WritePoolTotal int64 `json:"write_pool_total"` + PayedFees int64 `json:"payed_fees"` + CreatedAt time.Time + UpdatedAt time.Time } +type BlobberSnapshot struct { + BlobberID string `json:"id" gorm:"uniquIndex"` + URL string `json:"url"` + Round int64 `json:"round"` + WritePrice currency.Coin `json:"write_price"` + Capacity int64 `json:"capacity"` // total blobber capacity + Allocated int64 `json:"allocated"` // allocated capacity + SavedData int64 `json:"saved_data"` + ReadData int64 `json:"read_data"` + OffersTotal currency.Coin `json:"offers_total"` + TotalServiceCharge currency.Coin `json:"total_service_charge"` + TotalRewards currency.Coin `json:"total_rewards"` + TotalStake currency.Coin `json:"total_stake"` + TotalBlockRewards currency.Coin `json:"total_block_rewards"` + TotalStorageIncome currency.Coin `json:"total_storage_income"` + TotalReadIncome currency.Coin `json:"total_read_income"` + TotalSlashedStake currency.Coin `json:"total_slashed_stake"` + ChallengesPassed uint64 `json:"challenges_passed"` + ChallengesCompleted uint64 `json:"challenges_completed"` + OpenChallenges uint64 `json:"open_challenges"` + CreationRound int64 `json:"creation_round"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` +} + +type SharderSnapshot struct { + SharderID string `json:"id" gorm:"uniqueIndex"` + Round int64 `json:"round"` + URL string `json:"url"` -type MinerList struct { - Miners []Miner `json:"miners"` + Fees currency.Coin `json:"fees"` + TotalStake currency.Coin `json:"total_stake"` + TotalRewards currency.Coin `json:"total_rewards"` + ServiceCharge float64 `json:"service_charge"` + CreationRound int64 `json:"creation_round"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` } +type AuthorizerSnapshot struct { + AuthorizerID string `json:"id" gorm:"uniquIndex"` + Round int64 `json:"round"` + URL string `json:"url"` -type ValidatorList struct { - Validators []Validator `json:"validators"` + Fee currency.Coin `json:"fee"` + TotalStake currency.Coin `json:"total_stake"` + TotalRewards currency.Coin `json:"total_rewards"` + TotalMint currency.Coin `json:"total_mint"` + TotalBurn currency.Coin `json:"total_burn"` + ServiceCharge float64 `json:"service_charge"` + CreationRound int64 `json:"creation_round"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` } +type ValidatorSnapshot struct { + ValidatorID string `json:"id" gorm:"uniqueIndex"` + URL string `json:"url"` -type AuthorizerList struct { - Authorizers []Authorizer `json:"authorizers"` + Round int64 `json:"round"` + TotalStake currency.Coin `json:"total_stake"` + TotalRewards currency.Coin `json:"total_rewards"` + ServiceCharge float64 `json:"service_charge"` + CreationRound int64 `json:"creation_round"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` +} +type MinerSnapshot struct { + MinerID string `json:"id" gorm:"uniqueIndex"` + Round int64 `json:"round"` + URL string `json:"url"` + Fees currency.Coin `json:"fees"` + TotalStake currency.Coin `json:"total_stake"` + TotalRewards currency.Coin `json:"total_rewards"` + ServiceCharge float64 `json:"service_charge"` + BlocksFinalised int64 `json:"blocks_finalised"` + CreationRound int64 `json:"creation_round"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` } diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 4fd30ec5ec..689a0a12c7 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -2276,6 +2276,23 @@ func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) ( tableEntity = model.Validator{} case "sharder": tableEntity = model.Sharder{} + case "user": + tableEntity = model.User{} + case "user_snapshot": + tableEntity = model.UserSnapshot{} + case "miner_snapshot": + tableEntity = model.MinerSnapshot{} + case "blobber_snapshot": + tableEntity = model.BlobberSnapshot{} + case "validator_snapshot": + tableEntity = model.ValidatorSnapshot{} + case "sharder_snapshot": + tableEntity = model.SharderSnapshot{} + case "authorizer_snapshot": + tableEntity = model.AuthorizerSnapshot{} + case "provider_rewards": + tableEntity = model.ProviderRewards{} + } urlBuilder.queries.Set("entity", tableName) urlBuilder.queries.Set("fields", extractFields(tableEntity)) diff --git a/internal/api/util/client/zbox_client.go b/internal/api/util/client/zbox_client.go index 0e0f111077..ec43bbc9b9 100644 --- a/internal/api/util/client/zbox_client.go +++ b/internal/api/util/client/zbox_client.go @@ -1330,6 +1330,23 @@ func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) ([] tableEntity = model.Validator{} case "sharder": tableEntity = model.Sharder{} + case "user": + tableEntity = model.User{} + case "user_snapshot": + tableEntity = model.UserSnapshot{} + case "miner_snapshot": + tableEntity = model.MinerSnapshot{} + case "blobber_snapshot": + tableEntity = model.BlobberSnapshot{} + case "validator_snapshot": + tableEntity = model.ValidatorSnapshot{} + case "sharder_snapshot": + tableEntity = model.SharderSnapshot{} + case "authorizer_snapshot": + tableEntity = model.AuthorizerSnapshot{} + case "provider_rewards": + tableEntity = model.ProviderRewards{} + } urlBuilder.queries.Set("table", tableName) urlBuilder.queries.Set("fields", extractFields(tableEntity)) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 22167c44a3..f533319edb 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -2,49 +2,119 @@ package api_tests import ( "testing" + "time" "github.com/0chain/system_test/internal/api/util/test" + "github.com/0chain/system_test/internal/currency" "github.com/stretchr/testify/require" ) +func ParseToTimeIfValid(val interface{}) interface{} { + // check if val is a time.Time as a string + valStr, ok := val.(string) + if ok { + res, err := time.Parse(time.RFC3339Nano, valStr) + if err == nil { + // Parsing failed, v1Str is not a time string + return res + } + } + return val +} + +func CompareEntityTables(t *test.SystemTest, entity string) { + + entityTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, entity) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode()) + + entityTable_0box, resp, err := zboxClient.QueryDataFrom0box(t, entity) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode()) + + require.Equal(t, len(entityTable_Sharder), len(entityTable_0box)) + // t.Logf("entityTable_Sharder: %+v", entityTable_Sharder) + // t.Logf("entityTable_0box: %+v", entityTable_0box) + + var entity_Sharder, entity_0box map[string]map[string]interface{} + entity_Sharder = make(map[string]map[string]interface{}) + entity_0box = make(map[string]map[string]interface{}) + for i := 0; i < len(entityTable_Sharder); i++ { + e_sharder := entityTable_Sharder[i].(map[string]interface{}) + e_0box := entityTable_0box[i].(map[string]interface{}) + entity_Sharder[e_sharder["ID"].(string)] = e_sharder + entity_0box[e_0box["id"].(string)] = e_0box + } + for k, v := range entity_Sharder { + for k1, val1 := range v { + if val1 != nil && entity_0box[k][k1] != nil { + // how to know if v1 is a time.Time as a string + v1 := ParseToTimeIfValid(val1) + switch v1.(type) { + case float64: + require.InDelta(t, v1.(float64), entity_0box[k][k1].(float64), 0.05*v1.(float64)) + case string: + require.Equal(t, v1.(string), entity_0box[k][k1].(string)) + case bool: + require.Equal(t, v1.(bool), entity_0box[k][k1].(bool)) + case int: + require.InDelta(t, v1.(int), entity_0box[k][k1].(int), 0.05*float64(v1.(int))) + case int64: + require.InDelta(t, v1.(int64), entity_0box[k][k1].(int64), 0.05*float64(v1.(int64))) + case uint64: + require.InDelta(t, v1.(uint64), entity_0box[k][k1].(uint64), 0.05*float64(v1.(uint64))) + case time.Time: + t1 := v1.(time.Time) + entity_0box[k][k1] = ParseToTimeIfValid(entity_0box[k][k1]) + t2 := entity_0box[k][k1].(time.Time) + diff := t1.Sub(t2) + require.LessOrEqual(t, diff.Seconds(), 1000.0, "Time difference is more than 1000 seconds") + require.GreaterOrEqual(t, diff.Seconds(), -1000.0, "Time difference is more than -1000 seconds") + case currency.Coin: + t1, err1 := (v1.(currency.Coin)).Int64() + t2, err2 := entity_0box[k][k1].(currency.Coin).Int64() + require.NoError(t, err1) + require.NoError(t, err2) + require.InDelta(t, t1, t2, 0.05*float64(t1)) + + } + + } + } + + } +} + func TestCompares0boxTablesWithSharder(testSetup *testing.T) { t := test.NewSystemTest(testSetup) t.SetSmokeTests("Compare 0box tables with sharder tables") t.RunSequentially("Compare Miner tables", func(t *test.SystemTest) { + CompareEntityTables(t, "miner") + }) + t.RunSequentially("Compare Blobber tables", func(t *test.SystemTest) { + CompareEntityTables(t, "blobber") + }) + // t.RunSequentially("Compare Sharder tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "sharder") + // }) + // t.RunSequentially("Compare Validator tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "validator") + // }) + // t.RunSequentially("Compare Authorizer tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "authorizer") + // }) + // t.RunSequentially("Compare ProviderRewards tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "provider_rewards") + // }) + // t.RunSequentially("Compare User tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "user") + // }) + // t.RunSequentially("Compare Miner Snapshot tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "miner_snapshot") + // }) + // t.RunSequentially("Compare Blobber Snapshot tables", func(t *test.SystemTest) { + // CompareEntityTables(t, "blobber_snapshot") + // }) - minersTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, "miner") - require.NoError(t, err) - require.Equal(t, 200, resp.StatusCode()) - - minersTable_0box, resp, err := zboxClient.QueryDataFrom0box(t, "miner") - require.NoError(t, err) - require.Equal(t, 200, resp.StatusCode()) - - require.Equal(t, len(minersTable_Sharder), len(minersTable_0box)) - // t.Logf("minersTable_Sharder: %+v", minersTable_Sharder) - // t.Logf("minersTable_0box: %+v", minersTable_0box) - - var miners_Sharder, miners_0box map[string]map[string]interface{} - miners_Sharder = make(map[string]map[string]interface{}) - miners_0box = make(map[string]map[string]interface{}) - for i := 0; i < len(minersTable_Sharder); i++ { - m_sharder := minersTable_Sharder[i].(map[string]interface{}) - m_0box := minersTable_0box[i].(map[string]interface{}) - miners_Sharder[m_sharder["ID"].(string)] = m_sharder - miners_0box[m_0box["id"].(string)] = m_0box - } - for k, v := range miners_Sharder { - for k1, v1 := range v { - if v1 != nil && miners_0box[k][k1] != nil { - if v1 != miners_0box[k][k1] { - t.Logf("id:%s and key:%s", k, k1) - } - require.Equal(t, v1, miners_0box[k][k1]) - } - } - - } - }, - ) } From 24ab26e103cf46ab737e6a8db535b61c77ba9384 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Tue, 9 Jul 2024 01:55:09 +0530 Subject: [PATCH 07/10] remaining --- tests/api_tests/compare_0box_sharder_test.go | 42 ++++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index f533319edb..9f34504730 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -95,26 +95,26 @@ func TestCompares0boxTablesWithSharder(testSetup *testing.T) { t.RunSequentially("Compare Blobber tables", func(t *test.SystemTest) { CompareEntityTables(t, "blobber") }) - // t.RunSequentially("Compare Sharder tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "sharder") - // }) - // t.RunSequentially("Compare Validator tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "validator") - // }) - // t.RunSequentially("Compare Authorizer tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "authorizer") - // }) - // t.RunSequentially("Compare ProviderRewards tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "provider_rewards") - // }) - // t.RunSequentially("Compare User tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "user") - // }) - // t.RunSequentially("Compare Miner Snapshot tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "miner_snapshot") - // }) - // t.RunSequentially("Compare Blobber Snapshot tables", func(t *test.SystemTest) { - // CompareEntityTables(t, "blobber_snapshot") - // }) + t.RunSequentially("Compare Sharder tables", func(t *test.SystemTest) { + CompareEntityTables(t, "sharder") + }) + t.RunSequentially("Compare Validator tables", func(t *test.SystemTest) { + CompareEntityTables(t, "validator") + }) + t.RunSequentially("Compare Authorizer tables", func(t *test.SystemTest) { + CompareEntityTables(t, "authorizer") + }) + t.RunSequentially("Compare ProviderRewards tables", func(t *test.SystemTest) { + CompareEntityTables(t, "provider_rewards") + }) + t.RunSequentially("Compare User tables", func(t *test.SystemTest) { + CompareEntityTables(t, "user") + }) + t.RunSequentially("Compare Miner Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "miner_snapshot") + }) + t.RunSequentially("Compare Blobber Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "blobber_snapshot") + }) } From 89268e57d6f2b669bb8dc05145bccddb1cd7bd5b Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Tue, 9 Jul 2024 01:57:13 +0530 Subject: [PATCH 08/10] remaining --- tests/api_tests/compare_0box_sharder_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 9f34504730..f966219a98 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -116,5 +116,17 @@ func TestCompares0boxTablesWithSharder(testSetup *testing.T) { t.RunSequentially("Compare Blobber Snapshot tables", func(t *test.SystemTest) { CompareEntityTables(t, "blobber_snapshot") }) + t.RunSequentially("Compare Sharder Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "sharder_snapshot") + }) + t.RunSequentially("Compare Validator Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "validator_snapshot") + }) + t.RunSequentially("Compare Authorizer Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "authorizer_snapshot") + }) + t.RunSequentially("Compare User Snapshot tables", func(t *test.SystemTest) { + CompareEntityTables(t, "user_snapshot") + }) } From fe70d2952a36f0cb505b71ee2bf7b9b89a9205ef Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Tue, 9 Jul 2024 02:01:01 +0530 Subject: [PATCH 09/10] comment fix --- tests/api_tests/compare_0box_sharder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index f966219a98..489c280a9f 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -15,10 +15,10 @@ func ParseToTimeIfValid(val interface{}) interface{} { if ok { res, err := time.Parse(time.RFC3339Nano, valStr) if err == nil { - // Parsing failed, v1Str is not a time string return res } } + // Parsing failed, v1Str is not a time string return val } From 13bd2f96af25286bba761eff05f141df61e1e6a7 Mon Sep 17 00:00:00 2001 From: shohan2001 Date: Sun, 21 Jul 2024 20:55:48 +0530 Subject: [PATCH 10/10] adding final changes --- internal/api/model/compare.go | 186 ++++++------------- internal/api/model/zbox.go | 13 -- internal/api/util/client/api_client.go | 12 -- internal/api/util/client/zbox_client.go | 12 -- tests/api_tests/compare_0box_sharder_test.go | 69 ++++--- 5 files changed, 95 insertions(+), 197 deletions(-) diff --git a/internal/api/model/compare.go b/internal/api/model/compare.go index acd7ed117b..0673757a96 100644 --- a/internal/api/model/compare.go +++ b/internal/api/model/compare.go @@ -33,7 +33,6 @@ type Blobber struct { ChallengesPassed uint64 `json:"challenges_passed"` ChallengesCompleted uint64 `json:"challenges_completed"` OpenChallenges uint64 `json:"open_challenges"` - RankMetric float64 `json:"rank_metric"` TotalBlockRewards currency.Coin `json:"total_block_rewards"` TotalStorageIncome currency.Coin `json:"total_storage_income"` TotalReadIncome currency.Coin `json:"total_read_income"` @@ -67,57 +66,65 @@ type Miner struct { } type Sharder struct { - ID string `gorm:"primaryKey" json:"id"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates string `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` - TotalStake string `json:"total_stake"` - // Rewards ProviderRewards `json:"rewards" gorm:"foreignKey:ProviderID"` + ID string `json:"id"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates string `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake string `json:"total_stake"` Downtime uint64 `json:"downtime"` LastHealthCheck common.Timestamp `json:"last_health_check"` IsKilled bool `json:"is_killed"` IsShutdown bool `json:"is_shutdown"` - - N2NHost string `gorm:"column:n2n_host"` - Host string - Port int - Path string - PublicKey string - ShortName string - BuildTag string - Delete bool - Fees currency.Coin - Active bool - CreationRound int64 `json:"creation_round" gorm:"index:idx_sharder_creation_round"` + N2NHost string `json:"n2n_host"` + Host string `json:"host"` + Port int `json:"port"` + Path string `json:"path"` + PublicKey string `json:"public_key"` + ShortName string `json:"short_name"` + BuildTag string `json:"build_tag"` + Delete bool `json:"delete"` + Fees currency.Coin `json:"fees"` + Active bool `json:"active"` + CreationRound int64 `json:"creation_round" gorm:"index:idx_sharder_creation_round"` } type Validator struct { - ValidatorID string `json:"validator_id"` - BaseUrl string `json:"url"` - StakeTotal currency.Coin `json:"stake_total"` - PublicKey string `json:"public_key"` - LastHealthCheck common.Timestamp `json:"last_health_check"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` - TotalServiceCharge currency.Coin `json:"total_service_charge"` - UncollectedServiceCharge currency.Coin `json:"uncollected_service_charge"` + ID string `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake currency.Coin `json:"total_stake"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + Url string `json:"base_url"` + PublicKey string `json:"public_key"` + CreationRound int64 `json:"creation_round"` } type Authorizer struct { - AuthorizerID string `json:"id"` - URL string `json:"url"` - Fee currency.Coin `json:"fee"` - LastHealthCheck int64 `json:"last_health_check"` - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` + ID string `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` + TotalStake currency.Coin `json:"total_stake"` + Downtime uint64 `json:"downtime"` + LastHealthCheck common.Timestamp `json:"last_health_check"` + IsKilled bool `json:"is_killed"` + IsShutdown bool `json:"is_shutdown"` + URL string `json:"url"` + Fee currency.Coin `json:"fee"` + TotalMint currency.Coin `json:"total_mint"` + TotalBurn currency.Coin `json:"total_burn"` + CreationRound int64 `json:"creation_round"` } type User struct { - ID uint `json:"id" gorm:"primarykey"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` UserID string `json:"user_id" gorm:"uniqueIndex"` @@ -127,93 +134,12 @@ type User struct { Nonce int64 `json:"nonce"` MintNonce int64 `json:"mint_nonce"` } -type UserSnapshot struct { - UserID string `json:"user_id" gorm:"uniqueIndex"` - Round int64 `json:"round"` - TotalReward int64 `json:"total_reward"` - CollectedReward int64 `json:"collected_reward"` - TotalStake int64 `json:"total_stake"` - ReadPoolTotal int64 `json:"read_pool_total"` - WritePoolTotal int64 `json:"write_pool_total"` - PayedFees int64 `json:"payed_fees"` - CreatedAt time.Time - UpdatedAt time.Time -} -type BlobberSnapshot struct { - BlobberID string `json:"id" gorm:"uniquIndex"` - URL string `json:"url"` - Round int64 `json:"round"` - WritePrice currency.Coin `json:"write_price"` - Capacity int64 `json:"capacity"` // total blobber capacity - Allocated int64 `json:"allocated"` // allocated capacity - SavedData int64 `json:"saved_data"` - ReadData int64 `json:"read_data"` - OffersTotal currency.Coin `json:"offers_total"` - TotalServiceCharge currency.Coin `json:"total_service_charge"` - TotalRewards currency.Coin `json:"total_rewards"` - TotalStake currency.Coin `json:"total_stake"` - TotalBlockRewards currency.Coin `json:"total_block_rewards"` - TotalStorageIncome currency.Coin `json:"total_storage_income"` - TotalReadIncome currency.Coin `json:"total_read_income"` - TotalSlashedStake currency.Coin `json:"total_slashed_stake"` - ChallengesPassed uint64 `json:"challenges_passed"` - ChallengesCompleted uint64 `json:"challenges_completed"` - OpenChallenges uint64 `json:"open_challenges"` - CreationRound int64 `json:"creation_round"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` -} - -type SharderSnapshot struct { - SharderID string `json:"id" gorm:"uniqueIndex"` - Round int64 `json:"round"` - URL string `json:"url"` - - Fees currency.Coin `json:"fees"` - TotalStake currency.Coin `json:"total_stake"` - TotalRewards currency.Coin `json:"total_rewards"` - ServiceCharge float64 `json:"service_charge"` - CreationRound int64 `json:"creation_round"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` -} -type AuthorizerSnapshot struct { - AuthorizerID string `json:"id" gorm:"uniquIndex"` - Round int64 `json:"round"` - URL string `json:"url"` - - Fee currency.Coin `json:"fee"` - TotalStake currency.Coin `json:"total_stake"` - TotalRewards currency.Coin `json:"total_rewards"` - TotalMint currency.Coin `json:"total_mint"` - TotalBurn currency.Coin `json:"total_burn"` - ServiceCharge float64 `json:"service_charge"` - CreationRound int64 `json:"creation_round"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` -} -type ValidatorSnapshot struct { - ValidatorID string `json:"id" gorm:"uniqueIndex"` - URL string `json:"url"` - - Round int64 `json:"round"` - TotalStake currency.Coin `json:"total_stake"` - TotalRewards currency.Coin `json:"total_rewards"` - ServiceCharge float64 `json:"service_charge"` - CreationRound int64 `json:"creation_round"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` -} -type MinerSnapshot struct { - MinerID string `json:"id" gorm:"uniqueIndex"` - Round int64 `json:"round"` - URL string `json:"url"` - Fees currency.Coin `json:"fees"` - TotalStake currency.Coin `json:"total_stake"` - TotalRewards currency.Coin `json:"total_rewards"` - ServiceCharge float64 `json:"service_charge"` - BlocksFinalised int64 `json:"blocks_finalised"` - CreationRound int64 `json:"creation_round"` - IsKilled bool `json:"is_killed"` - IsShutdown bool `json:"is_shutdown"` +type ProviderRewards struct { + ID uint `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ProviderID string `json:"provider_id"` + Rewards currency.Coin `json:"rewards"` + TotalRewards currency.Coin `json:"total_rewards"` + RoundServiceChargeLastUpdated int64 `json:"round_service_charge_last_updated"` } diff --git a/internal/api/model/zbox.go b/internal/api/model/zbox.go index 46c1340ec6..ffdbd2aa8a 100644 --- a/internal/api/model/zbox.go +++ b/internal/api/model/zbox.go @@ -1,9 +1,6 @@ package model import ( - "time" - - "github.com/0chain/common/core/currency" "github.com/0chain/gosdk/core/common" "github.com/0chain/system_test/internal/api/util/test" resty "github.com/go-resty/resty/v2" @@ -228,15 +225,5 @@ type ReferralRankOfUser struct { ReferrerID int64 `json:"referrer_id"` } -type ProviderRewards struct { - ID uint `json:"id" gorm:"primarykey"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ProviderID string `json:"provider_id" gorm:"uniqueIndex"` - Rewards currency.Coin `json:"rewards"` - TotalRewards currency.Coin `json:"total_rewards"` - RoundServiceChargeLastUpdated int64 `json:"round_service_charge_last_updated"` -} - type ZboxGraphEndpoint func(*test.SystemTest, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) type ZboxGraphBlobberEndpoint func(*test.SystemTest, string, *ZboxGraphRequest) (*ZboxGraphInt64Response, *resty.Response, error) diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 689a0a12c7..5c61ccc502 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -2278,18 +2278,6 @@ func (c *APIClient) QueryDataFromSharder(t *test.SystemTest, tableName string) ( tableEntity = model.Sharder{} case "user": tableEntity = model.User{} - case "user_snapshot": - tableEntity = model.UserSnapshot{} - case "miner_snapshot": - tableEntity = model.MinerSnapshot{} - case "blobber_snapshot": - tableEntity = model.BlobberSnapshot{} - case "validator_snapshot": - tableEntity = model.ValidatorSnapshot{} - case "sharder_snapshot": - tableEntity = model.SharderSnapshot{} - case "authorizer_snapshot": - tableEntity = model.AuthorizerSnapshot{} case "provider_rewards": tableEntity = model.ProviderRewards{} diff --git a/internal/api/util/client/zbox_client.go b/internal/api/util/client/zbox_client.go index ec43bbc9b9..dff2ae1cc0 100644 --- a/internal/api/util/client/zbox_client.go +++ b/internal/api/util/client/zbox_client.go @@ -1332,18 +1332,6 @@ func (c *ZboxClient) QueryDataFrom0box(t *test.SystemTest, tableName string) ([] tableEntity = model.Sharder{} case "user": tableEntity = model.User{} - case "user_snapshot": - tableEntity = model.UserSnapshot{} - case "miner_snapshot": - tableEntity = model.MinerSnapshot{} - case "blobber_snapshot": - tableEntity = model.BlobberSnapshot{} - case "validator_snapshot": - tableEntity = model.ValidatorSnapshot{} - case "sharder_snapshot": - tableEntity = model.SharderSnapshot{} - case "authorizer_snapshot": - tableEntity = model.AuthorizerSnapshot{} case "provider_rewards": tableEntity = model.ProviderRewards{} diff --git a/tests/api_tests/compare_0box_sharder_test.go b/tests/api_tests/compare_0box_sharder_test.go index 489c280a9f..a48d05d10e 100644 --- a/tests/api_tests/compare_0box_sharder_test.go +++ b/tests/api_tests/compare_0box_sharder_test.go @@ -22,7 +22,7 @@ func ParseToTimeIfValid(val interface{}) interface{} { return val } -func CompareEntityTables(t *test.SystemTest, entity string) { +func CompareEntityTables(t *test.SystemTest, entity string, id_sharder string, id_0box string) { entityTable_Sharder, resp, err := apiClient.QueryDataFromSharder(t, entity) require.NoError(t, err) @@ -32,6 +32,33 @@ func CompareEntityTables(t *test.SystemTest, entity string) { require.NoError(t, err) require.Equal(t, 200, resp.StatusCode()) + if entity == "user" { + entityTable_SharderCopy := entityTable_Sharder + entityTable_Sharder = entityTable_Sharder[:0] + // t.Logf("entityTable_Sharder: %+v", entityTable_Sharder) + + for i := 0; i < len(entityTable_SharderCopy); i++ { + // t.Logf("entityTable_SharderCopy[i].(map[string]interface{})[\"round\"]: %+v", entityTable_SharderCopy[i].(map[string]interface{})["round"]) + temp_copy := entityTable_SharderCopy[i].(map[string]interface{}) + val := temp_copy["round"] + if val != nil { + switch val.(type) { + case float64: + if val.(float64) != 0 { + entityTable_Sharder = append(entityTable_Sharder, temp_copy) + } + case int: + if val.(int) != 0 { + entityTable_Sharder = append(entityTable_Sharder, temp_copy) + } + case int64: + if val.(int64) != 0 { + entityTable_Sharder = append(entityTable_Sharder, temp_copy) + } + } + } + } + } require.Equal(t, len(entityTable_Sharder), len(entityTable_0box)) // t.Logf("entityTable_Sharder: %+v", entityTable_Sharder) // t.Logf("entityTable_0box: %+v", entityTable_0box) @@ -42,17 +69,17 @@ func CompareEntityTables(t *test.SystemTest, entity string) { for i := 0; i < len(entityTable_Sharder); i++ { e_sharder := entityTable_Sharder[i].(map[string]interface{}) e_0box := entityTable_0box[i].(map[string]interface{}) - entity_Sharder[e_sharder["ID"].(string)] = e_sharder - entity_0box[e_0box["id"].(string)] = e_0box + entity_Sharder[e_sharder[id_sharder].(string)] = e_sharder + entity_0box[e_0box[id_0box].(string)] = e_0box } for k, v := range entity_Sharder { for k1, val1 := range v { if val1 != nil && entity_0box[k][k1] != nil { - // how to know if v1 is a time.Time as a string + // how to know if v1 is a time.Time as a stringaf v1 := ParseToTimeIfValid(val1) switch v1.(type) { case float64: - require.InDelta(t, v1.(float64), entity_0box[k][k1].(float64), 0.05*v1.(float64)) + require.InDelta(t, v1.(float64), entity_0box[k][k1].(float64), 0.05*v1.(float64), "Float64 values are not equal for field %s", k1) case string: require.Equal(t, v1.(string), entity_0box[k][k1].(string)) case bool: @@ -90,43 +117,25 @@ func TestCompares0boxTablesWithSharder(testSetup *testing.T) { t.SetSmokeTests("Compare 0box tables with sharder tables") t.RunSequentially("Compare Miner tables", func(t *test.SystemTest) { - CompareEntityTables(t, "miner") + CompareEntityTables(t, "miner", "ID", "id") }) t.RunSequentially("Compare Blobber tables", func(t *test.SystemTest) { - CompareEntityTables(t, "blobber") + CompareEntityTables(t, "blobber", "ID", "id") }) t.RunSequentially("Compare Sharder tables", func(t *test.SystemTest) { - CompareEntityTables(t, "sharder") + CompareEntityTables(t, "sharder", "ID", "id") }) t.RunSequentially("Compare Validator tables", func(t *test.SystemTest) { - CompareEntityTables(t, "validator") + CompareEntityTables(t, "validator", "ID", "id") }) t.RunSequentially("Compare Authorizer tables", func(t *test.SystemTest) { - CompareEntityTables(t, "authorizer") + CompareEntityTables(t, "authorizer", "ID", "id") }) t.RunSequentially("Compare ProviderRewards tables", func(t *test.SystemTest) { - CompareEntityTables(t, "provider_rewards") + CompareEntityTables(t, "provider_rewards", "provider_id", "provider_id") }) t.RunSequentially("Compare User tables", func(t *test.SystemTest) { - CompareEntityTables(t, "user") - }) - t.RunSequentially("Compare Miner Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "miner_snapshot") - }) - t.RunSequentially("Compare Blobber Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "blobber_snapshot") - }) - t.RunSequentially("Compare Sharder Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "sharder_snapshot") - }) - t.RunSequentially("Compare Validator Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "validator_snapshot") - }) - t.RunSequentially("Compare Authorizer Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "authorizer_snapshot") - }) - t.RunSequentially("Compare User Snapshot tables", func(t *test.SystemTest) { - CompareEntityTables(t, "user_snapshot") + CompareEntityTables(t, "user", "txn_hash", "txn_hash") }) }