Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix] variable change & game rating DAG fix #112

Merged
merged 1 commit into from
Feb 27, 2024
Merged

[fix] variable change & game rating DAG fix #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions dags/game/game_ccu_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def save_to_json(data, collect_time):
"on_failure_callback": slack.on_failure_callback,
},
) as dag:

bucket_name = "de-2-1-bucket"

current_time = "{{ data_interval_end.in_timezone('Asia/Seoul') }}"
Expand All @@ -97,7 +97,7 @@ def save_to_json(data, collect_time):
data = get_ccu()
collect_time = f"{year}-{month}-{day} {hour}:{minutes}"
data_json = save_to_json(data, collect_time)

task_load_raw_data = S3CreateObjectOperator(
task_id="create_object",
s3_bucket=bucket_name,
Expand Down
18 changes: 14 additions & 4 deletions dags/game/game_rating_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import re
import requests
import json
import time


def connect_to_mysql():
Expand All @@ -23,12 +24,21 @@ def connect_to_mysql():
# Game info 테이블에 있는 Game들의 app_id를 이용해 게임 정량 평가를 가져오는 함수
def get_rating(app_id):
url = f"https://store.steampowered.com/app/{app_id}"
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")

reviewdesc_short = soup.find_all("span", {"class": "responsive_reviewdesc_short"})
for i in range(3):
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")

if reviewdesc_short == []:
reviewdesc_short = soup.find_all(
"span", {"class": "responsive_reviewdesc_short"}
)

if reviewdesc_short:
break

print(f"Retry {i+1} times for {app_id}")
time.sleep(90)
else:
return {
"game_id": app_id,
"all_positive_num": 0,
Expand Down
1 change: 1 addition & 0 deletions dags/streaming/follower.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def delete_file(file_path):
os.remove(file_path)
# print(f"Deleted {file_path}")


# when change file_names => plz check delete func!!
def delete_files(**kwargs):
# 플랫폼별로 파일 삭제
Expand Down
2 changes: 1 addition & 1 deletion dataCollector/glue&athena/glue_game_ccu_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
# 최상위 레벨의 key를 중심으로 explode하기
df = game_ccu_datasource.select(
explode(game_ccu_datasource.raw_game_ccu).alias("raw_game_ccu"),
col("collect_time").alias("COLLECT_TIME")
col("collect_time").alias("COLLECT_TIME"),
)

df = df.select(
Expand Down
2 changes: 1 addition & 1 deletion dataCollector/glue&athena/glue_game_rating_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@

df = df.select(
col("raw_game_rating.game_id").alias("GAME_ID"),
lit("{{ collect_time }}").alias("COLLECT_DATE"),
lit("{{ collect_date }}").alias("COLLECT_DATE"),
col("raw_game_rating.recent_positive_num").alias("RECENT_POSITIVE_NUM"),
col("raw_game_rating.recent_positive_percent").alias("RECENT_POSITIVE_PERCENT"),
col("raw_game_rating.all_positive_num").alias("ALL_POSITIVE_NUM"),
Expand Down