From cb0c79df7dbae5dbe1484a24edf87b58b12534f8 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Wed, 30 Oct 2024 18:53:43 +0900 Subject: [PATCH] Fix duplicate ranking insert --- tests/conftest.py | 8 ++++---- world_boss/app/raid.py | 13 ++++++------- world_boss/app/tasks.py | 28 +++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e44a6e..6d07ae6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -716,7 +716,7 @@ def fx_ranking_reward_csv() -> str: 11,124,0x74C6aFdB7405e8109D63521Fd8363f25fead0EcA,03b16b0096F8d5Df707c7cDCCf295a9A78182EA8,370,RUNESTONE_FENRIR1,0,2993 11,124,0x74C6aFdB7405e8109D63521Fd8363f25fead0EcA,03b16b0096F8d5Df707c7cDCCf295a9A78182EA8,105,RUNESTONE_FENRIR2,0,2993 11,124,0x74C6aFdB7405e8109D63521Fd8363f25fead0EcA,03b16b0096F8d5Df707c7cDCCf295a9A78182EA8,25,RUNESTONE_FENRIR3,0,2993 -11,125,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,500000,CRYSTAL,18,2993 -11,125,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,370,RUNESTONE_FENRIR1,0,2993 -11,125,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,105,RUNESTONE_FENRIR2,0,2993 -11,125,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,25,RUNESTONE_FENRIR3,0,2993""" +11,124,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,500000,CRYSTAL,18,2993 +11,124,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,370,RUNESTONE_FENRIR1,0,2993 +11,124,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,105,RUNESTONE_FENRIR2,0,2993 +11,124,0xB7Bc6def28325784aF2Ce0f5BDbD01F619A11DB8,25B24f81Cd6da3433e337857Fcbdd8b0ae43DF69,25,RUNESTONE_FENRIR3,0,2993""" diff --git a/world_boss/app/raid.py b/world_boss/app/raid.py index 3fbb7f5..1f3d136 100644 --- a/world_boss/app/raid.py +++ b/world_boss/app/raid.py @@ -456,9 +456,6 @@ def bulk_insert_transactions( tx_ids[n] = tx_id db.execute(insert(Transaction), tx_values) raid_id = int(rows[0][0]) - exist_rankings = [ - r for r, in db.query(WorldBossReward.ranking).filter_by(raid_id=raid_id) - ] # avatar_address : list of world boss reward amount world_boss_reward_amounts: dict[str, list[dict]] = {} # raid_id,ranking,agent_address,avatar_address,amount,ticker,decimal_places,target_nonce @@ -473,7 +470,7 @@ def bulk_insert_transactions( nonce = int(row[7]) # get or create world_boss_reward - if ranking not in exist_rankings and not world_boss_rewards.get(avatar_address): + if not world_boss_rewards.get(avatar_address): world_boss_reward = { "raid_id": raid_id, "ranking": ranking, @@ -493,10 +490,12 @@ def bulk_insert_transactions( world_boss_reward_amounts[avatar_address] = [] world_boss_reward_amounts[avatar_address].append(world_boss_reward_amount) if world_boss_rewards: - db.execute(insert(WorldBossReward), world_boss_rewards.values()) - result = db.query(WorldBossReward).filter_by(raid_id=raid_id) + result = db.execute( + insert(WorldBossReward).returning(WorldBossReward), + world_boss_rewards.values(), + ) values = [] - for reward in result: + for (reward,) in result: exist_tickers = [i.ticker for i in reward.amounts] if world_boss_rewards.get(reward.avatar_address): for amounts in world_boss_reward_amounts[reward.avatar_address]: diff --git a/world_boss/app/tasks.py b/world_boss/app/tasks.py index cd77c2c..4a86bc2 100644 --- a/world_boss/app/tasks.py +++ b/world_boss/app/tasks.py @@ -247,12 +247,12 @@ def check_season(): with TaskSessionLocal() as db: raid_id = get_latest_raid_id(db) total_count = data_provider_client.get_total_users_count(raid_id) - sync_count = offset = get_reward_count(db, raid_id) + sync_count = get_reward_count(db, raid_id) + offset = 0 # 최신 시즌 동기화 처리 if sync_count == total_count: upload_tx_list(raid_id) raid_id += 1 - offset = 0 save_ranking_rewards( raid_id=raid_id, payload_size=500, recipients_size=50, offset=offset ) @@ -272,11 +272,27 @@ def save_ranking_rewards( results: List[RankingRewardWithAgentDictionary] = [] time_stamp = get_next_month_last_day() memo = "world boss ranking rewards by world boss signer" + target_avatar_addresses = [] with TaskSessionLocal() as db: start_nonce = get_next_tx_nonce(db) - result = data_provider_client.get_ranking_rewards( - raid_id, NetworkType.MAIN, offset, payload_size - ) + while True: + result = data_provider_client.get_ranking_rewards( + raid_id, NetworkType.MAIN, offset, payload_size + ) + exist_avatar_addresses = [ + i + for i, in db.query(WorldBossReward.avatar_address).filter_by( + raid_id=raid_id + ) + ] + avatar_addresses = [r["raider"]["address"] for r in result] + target_avatar_addresses = [ + i for i in avatar_addresses if i not in exist_avatar_addresses + ] + if len(target_avatar_addresses) > 0: + break + else: + offset += payload_size rewards = update_agent_address( result, raid_id, NetworkType.MAIN, offset, payload_size ) @@ -288,6 +304,8 @@ def save_ranking_rewards( raider: RaiderWithAgentDictionary = r["raider"] ranking = raider["ranking"] avatar_address = raider["address"] + if avatar_address not in target_avatar_addresses: + continue reward_dict_list: List[RewardDictionary] = r["rewards"] for reward_dict in reward_dict_list: nonce = start_nonce + int(i / recipients_size)