Merge pull request #106 from zizzic/feature/glue

Feature/glue
zizzic · Feb 26, 2024 · 75a39bc · 75a39bc
2 parents 0b17f3f + c1c0abe
commit 75a39bc
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 10 deletions.
diff --git a/dataCollector/glue&athena/glue_template.py b/dataCollector/glue&athena/glue_template.py
@@ -22,6 +22,14 @@
     format="json",
     transformation_ctx="datasource",
 )
+# <---------add functions----------->
+
+# "PLATFORM" 컬럼을 기준으로 파티션을 구성
+partitioned_df = result_df.repartition("PLATFORM")
+
+# 파티션된 Spark DataFrame을 DynamicFrame으로 변환
+partitioned_dynamic_frame = DynamicFrame.fromDF(partitioned_df, glueContext, "partitioned_dynamic_frame")
+
 
 # Parquet으로 변환하여 S3에 저장
 glueContext.write_dynamic_frame.from_options(
@@ -31,7 +39,3 @@
     format="parquet",
     transformation_ctx="datasource",
 )
-
-
-# Job Bookmark의 상태를 최종적으로 커밋
-job.commit()
diff --git a/dataCollector/glue&athena/live_viewer_template.py b/dataCollector/glue&athena/live_viewer_template.py
@@ -73,17 +73,18 @@
 print("Schema Information:")
 result_df.printSchema()
 
-dynamicframe = DynamicFrame.fromDF(result_df, glueContext, "dynamicframe")
+# "PLATFORM" 컬럼을 기준으로 파티션을 구성
+partitioned_df = result_df.repartition("PLATFORM")
+
+# 파티션된 Spark DataFrame을 DynamicFrame으로 변환
+partitioned_dynamic_frame = DynamicFrame.fromDF(partitioned_df, glueContext, "partitioned_dynamic_frame")
 
 
 # Parquet으로 변환하여 S3에 저장
 glueContext.write_dynamic_frame.from_options(
-    frame=dynamicframe,
+    frame=partitioned_dynamic_frame,
     connection_type="s3",
     connection_options={"path": "{{ output_path }}"},
     format="parquet",
     transformation_ctx="datasource",
-)
-
-# Job Bookmark의 상태를 최종적으로 커밋
-job.commit()
+)