Add more tables to data warehouse

forklift/forklift/pipeline/flow_schedules/.~lock.sync_table_with_pandas.csv#

@@ -0,0 +1 @@
+Vincent Chéry,vincentchery,vincentchery-ThinkPad-T590,28.01.2025 17:47,file:///home/vincentchery/.config/libreoffice/4;

forklift/forklift/pipeline/flow_schedules/sync_table_from_db_connection.csv

@@ -1,3 +1,7 @@
 "source_database","source_table","query_filepath","destination_database","destination_table","ddl_script_path","order_by","cron_string"
 "monitorfish_proxy","control_objectives",,"monitorfish","control_objectives",,"year","30 4 * * *"
 "monitorfish_proxy","fleet_segments",,"monitorfish","fleet_segments",,"year","32 4 * * *"
+"monitorfish_proxy","ports",,"monitorfish","ports",,"locode","34 4 * * *"
+"monitorfish_proxy","species",,"monitorfish","species",,"species_code","36 4 * * *"
+"monitorfish_proxy","vessels",,"monitorfish","vessels",,"id","40 4 * * *"
+"monitorfish_proxy","infractions",,"monitorfish","infractions",,"natinf_code","42 4 * * *"

forklift/forklift/pipeline/flow_schedules/sync_table_with_pandas.csv

@@ -2,3 +2,7 @@
 "monitorenv_remote",,"public","amp_cacem","geopandas","geom","monitorenv","amp_cacem_tmp","monitorenv/create_amp_cacem_tmp.sql","post_process_amp_cacem.sql","amp_cacem","31 1 * * *"
 "monitorenv_remote","monitorenv_remote/analytics_actions.sql",,,"pandas","geom","monitorenv","analytics_actions","monitorenv/create_analytics_actions.sql",,,"30 * * * *"
 "monitorfish_remote","monitorfish_remote/analytics_controls_full_data.sql",,,"pandas","geom","monitorfish","analytics_controls_full_data","monitorfish/create_analytics_controls_full_data.sql",,,"32 1 * * *"
+"monitorfish_remote",,"public","fao_areas","geopandas","wkb_geometry","monitorfish","fao_areas_tmp","monitorfish/create_fao_areas_tmp.sql","post_process_fao_areas.sql","fao_areas","34 1 * * *"
+"monitorfish_remote",,"public","rectangles_stat_areas","geopandas","wkb_geometry","monitorfish","rectangles_stat_areas_tmp","monitorfish/create_rectangles_stat_areas_tmp.sql","post_process_rectangles_stat_areas.sql","rectangles_stat_areas","36 1 * * *"
+"monitorfish_remote","monitorfish_remote/facade_areas_subdivided.sql",,,"geopandas","geometry","monitorfish","facade_areas_subdivided_tmp","monitorfish/create_facade_areas_subdivided_tmp.sql","post_process_facade_areas_subdivided.sql","facade_areas_subdivided","38 1 * * *"
+"monitorfish_remote","monitorfish_remote/eez_areas.sql",,,"geopandas","wkb_geometry","monitorfish","eez_areas_tmp","monitorfish/create_eez_areas_tmp.sql","post_process_eez_areas.sql","eez_areas","40 1 * * *"

forklift/forklift/pipeline/queries/monitorfish_remote/eez_areas.sql

@@ -0,0 +1,10 @@
+SELECT
+    ogc_fid,
+    wkb_geometry,
+    "union",
+    territory1,
+    iso_ter1,
+    sovereign1,
+    iso_sov1,
+    area_km2
+FROM eez_areas

forklift/forklift/pipeline/queries/monitorfish_remote/facade_areas_subdivided.sql

@@ -0,0 +1,5 @@
+SELECT
+    facade,
+    geometry,
+    id
+FROM facade_areas_subdivided

forklift/forklift/pipeline/sql_scripts/data_flows/post_process_eez_areas.sql

@@ -0,0 +1,6 @@
+CREATE TABLE monitorfish.eez_areas ENGINE MergeTree ORDER BY iso_sov1 AS 
+SELECT
+    * EXCEPT(wkb_geometry),
+    readWKTMultiPolygon(wkb_geometry) AS wkb_geometry
+FROM monitorfish.eez_areas_tmp
+ORDER BY iso_sov1

forklift/forklift/pipeline/sql_scripts/data_flows/post_process_facade_areas_subdivided.sql

@@ -0,0 +1,6 @@
+CREATE TABLE monitorfish.facade_areas_subdivided ENGINE MergeTree ORDER BY facade AS 
+SELECT
+    * EXCEPT(geometry),
+    readWKTPolygon(geometry) AS geometry
+FROM monitorfish.facade_areas_subdivided_tmp
+ORDER BY facade

forklift/forklift/pipeline/sql_scripts/data_flows/post_process_fao_areas.sql

@@ -0,0 +1,6 @@
+CREATE TABLE monitorfish.fao_areas ENGINE MergeTree ORDER BY f_code AS 
+SELECT
+    * EXCEPT(wkb_geometry),
+    readWKTMultiPolygon(wkb_geometry) AS wkb_geometry
+FROM monitorfish.fao_areas_tmp
+ORDER BY f_code

forklift/forklift/pipeline/sql_scripts/data_flows/post_process_rectangles_stat_areas.sql

@@ -0,0 +1,6 @@
+CREATE TABLE monitorfish.rectangles_stat_areas ENGINE MergeTree ORDER BY id AS 
+SELECT
+    * EXCEPT(wkb_geometry),
+    readWKTMultiPolygon(wkb_geometry) AS wkb_geometry
+FROM monitorfish.rectangles_stat_areas_tmp
+ORDER BY id

forklift/forklift/pipeline/sql_scripts/ddl/monitorfish/create_eez_areas_tmp.sql

@@ -0,0 +1,12 @@
+CREATE TABLE {database:Identifier}.{table:Identifier} (
+    ogc_fid Int32,
+    wkb_geometry String,
+    "union" Nullable(String),
+    territory1 String,
+    iso_ter1 Nullable(String),
+    sovereign1 Nullable(String),
+    iso_sov1 String,
+    area_km2 Nullable(Float64)
+)
+ENGINE MergeTree
+ORDER BY iso_sov1

forklift/forklift/pipeline/sql_scripts/ddl/monitorfish/create_facade_areas_subdivided_tmp.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {database:Identifier}.{table:Identifier} (
+    facade LowCardinality(String),
+    geometry String,
+    id Int32
+)
+ENGINE MergeTree
+ORDER BY facade

forklift/forklift/pipeline/sql_scripts/ddl/monitorfish/create_fao_areas_tmp.sql

@@ -0,0 +1,18 @@
+CREATE TABLE {database:Identifier}.{table:Identifier} (
+    wkb_geometry String,
+    f_code String,
+    f_level Nullable(String),
+    f_status Nullable(Float64),
+    ocean Nullable(String),
+    subocean Nullable(String),
+    f_area Nullable(String),
+    f_subarea Nullable(String),
+    f_division Nullable(String),
+    f_subdivis Nullable(String),
+    f_subunit Nullable(String),
+    name_en Nullable(String),
+    name_fr Nullable(String),
+    name_es Nullable(String)
+)
+ENGINE MergeTree
+ORDER BY f_code

forklift/forklift/pipeline/sql_scripts/ddl/monitorfish/create_rectangles_stat_areas_tmp.sql

@@ -0,0 +1,13 @@
+CREATE TABLE {database:Identifier}.{table:Identifier} (
+    ogc_fid Int32,
+    wkb_geometry String,
+    id Float64,
+    icesname Nullable(String),
+    south Nullable(Float64),
+    west Nullable(Float64),
+    north Nullable(Float64),
+    east Nullable(Float64),
+    area_km2 Nullable(Float64)
+)
+ENGINE MergeTree
+ORDER BY id

forklift/tests/test_data/monitorfish/remote_database/V777.01__dummy_rectangles_stat_areas.sql

@@ -0,0 +1,23 @@
+INSERT INTO public.rectangles_stat_areas VALUES
+    (1, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000000424000000000000046C0000000000040424000000000008045C0000000000040424000000000008045C0000000000000424000000000000046C00000000000004240', 1, '01A0', 36.0000000000000000000000000000000, -44.0000000000000000000000000000000, 36.5000000000000000000000000000000, -43.0000000000000000000000000000000, 4987),
+    (2, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000040424000000000000046C0000000000080424000000000008045C0000000000080424000000000008045C0000000000040424000000000000046C00000000000404240', 2, '02A0', 36.5000000000000000000000000000000, -44.0000000000000000000000000000000, 37.0000000000000000000000000000000, -43.0000000000000000000000000000000, 4955),
+    (3, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000080424000000000000046C00000000000C0424000000000008045C00000000000C0424000000000008045C0000000000080424000000000000046C00000000000804240', 3, '03A0', 37.0000000000000000000000000000000, -44.0000000000000000000000000000000, 37.5000000000000000000000000000000, -43.0000000000000000000000000000000, 4923),
+    (4, '0106000020E6100000010000000103000000010000000500000000000000000046C00000000000C0424000000000000046C0000000000000434000000000008045C0000000000000434000000000008045C00000000000C0424000000000000046C00000000000C04240', 4, '04A0', 37.5000000000000000000000000000000, -44.0000000000000000000000000000000, 38.0000000000000000000000000000000, -43.0000000000000000000000000000000, 4891),
+    (5, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000000434000000000000046C0000000000040434000000000008045C0000000000040434000000000008045C0000000000000434000000000000046C00000000000004340', 5, '05A0', 38.0000000000000000000000000000000, -44.0000000000000000000000000000000, 38.5000000000000000000000000000000, -43.0000000000000000000000000000000, 4858),
+    (6, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000040434000000000000046C0000000000080434000000000008045C0000000000080434000000000008045C0000000000040434000000000000046C00000000000404340', 6, '06A0', 38.5000000000000000000000000000000, -44.0000000000000000000000000000000, 39.0000000000000000000000000000000, -43.0000000000000000000000000000000, 4825),
+    (7, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000080434000000000000046C00000000000C0434000000000008045C00000000000C0434000000000008045C0000000000080434000000000000046C00000000000804340', 7, '07A0', 39.0000000000000000000000000000000, -44.0000000000000000000000000000000, 39.5000000000000000000000000000000, -43.0000000000000000000000000000000, 4792),
+    (8, '0106000020E6100000010000000103000000010000000500000000000000000046C00000000000C0434000000000000046C0000000000000444000000000008045C0000000000000444000000000008045C00000000000C0434000000000000046C00000000000C04340', 8, '08A0', 39.5000000000000000000000000000000, -44.0000000000000000000000000000000, 40.0000000000000000000000000000000, -43.0000000000000000000000000000000, 4758),
+    (9, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000000444000000000000046C0000000000040444000000000008045C0000000000040444000000000008045C0000000000000444000000000000046C00000000000004440', 9, '09A0', 40.0000000000000000000000000000000, -44.0000000000000000000000000000000, 40.5000000000000000000000000000000, -43.0000000000000000000000000000000, 4724),
+    (10, '0106000020E6100000010000000103000000010000000500000000000000000046C0000000000040444000000000000046C0000000000080444000000000008045C0000000000080444000000000008045C0000000000040444000000000000046C00000000000404440', 10, '10A0', 40.5000000000000000000000000000000, -44.0000000000000000000000000000000, 41.0000000000000000000000000000000, -43.0000000000000000000000000000000, 4689),
+    (10999, '0106000020E610000001000000010300000001000000050000000000000000005140000000000080474000000000000051400000000000C0474000000000004051400000000000C047400000000000405140000000000080474000000000000051400000000000804740', 10999, '23M8', 47.0000000000000000000000000000000, 68.0000000000000000000000000000000, 47.5000000000000000000000000000000, 69.0000000000000000000000000000000, 4208),
+    (11000, '0106000020E6100000010000000103000000010000000500000000000000000051400000000000C04740000000000000514000000000000048400000000000405140000000000000484000000000004051400000000000C0474000000000000051400000000000C04740', 11000, '24M8', 47.5000000000000000000000000000000, 68.0000000000000000000000000000000, 48.0000000000000000000000000000000, 69.0000000000000000000000000000000, 4169),
+    (11001, '0106000020E610000001000000010300000001000000050000000000000000005140000000000000484000000000000051400000000000404840000000000040514000000000004048400000000000405140000000000000484000000000000051400000000000004840', 11001, '25M8', 48.0000000000000000000000000000000, 68.0000000000000000000000000000000, 48.5000000000000000000000000000000, 69.0000000000000000000000000000000, 4129),
+    (11002, '0106000020E610000001000000010300000001000000050000000000000000005140000000000040484000000000000051400000000000804840000000000040514000000000008048400000000000405140000000000040484000000000000051400000000000404840', 11002, '26M8', 48.5000000000000000000000000000000, 68.0000000000000000000000000000000, 49.0000000000000000000000000000000, 69.0000000000000000000000000000000, 4089),
+    (11003, '0106000020E610000001000000010300000001000000050000000000000000005140000000000080484000000000000051400000000000C0484000000000004051400000000000C048400000000000405140000000000080484000000000000051400000000000804840', 11003, '27M8', 49.0000000000000000000000000000000, 68.0000000000000000000000000000000, 49.5000000000000000000000000000000, 69.0000000000000000000000000000000, 4048),
+    (11401, '0106000020E6100000010000000103000000010000000500000000000000004050400000000000404AC000000000008050400000000000404AC000000000008050400000000000804AC000000000004050400000000000804AC000000000004050400000000000404AC0', 213, '474', NULL, NULL, NULL, NULL, NULL),
+    (11402, '0106000020E6100000010000000103000000010000000500000000000000008050400000000000404AC00000000000C050400000000000404AC00000000000C050400000000000804AC000000000008050400000000000804AC000000000008050400000000000404AC0', 214, '473', NULL, NULL, NULL, NULL, NULL),
+    (11403, '0106000020E610000001000000010300000001000000050000000000000000C050400000000000404AC000000000000051400000000000404AC000000000000051400000000000804AC00000000000C050400000000000804AC00000000000C050400000000000404AC0', 215, '472', NULL, NULL, NULL, NULL, NULL),
+    (11404, '0106000020E6100000010000000103000000010000000500000000000000000051400000000000404AC000000000004051400000000000404AC000000000004051400000000000804AC000000000000051400000000000804AC000000000000051400000000000404AC0', 216, '471', NULL, NULL, NULL, NULL, NULL),
+    (11405, '0106000020E6100000010000000103000000010000000500000000000000004050400000000000804AC000000000008050400000000000804AC000000000008050400000000000C04AC000000000004050400000000000C04AC000000000004050400000000000804AC0', 227, '484', NULL, NULL, NULL, NULL, NULL),
+    (11406, '0106000020E6100000010000000103000000010000000500000000000000008050400000000000804AC00000000000C050400000000000804AC00000000000C050400000000000C04AC000000000008050400000000000C04AC000000000008050400000000000804AC0', 228, '483', NULL, NULL, NULL, NULL, NULL),
+    (11407, '0106000020E610000001000000010300000001000000050000000000000000C050400000000000804AC000000000000051400000000000804AC000000000000051400000000000C04AC00000000000C050400000000000C04AC00000000000C050400000000000804AC0', 229, '482', NULL, NULL, NULL, NULL, NULL);  

forklift/tests/test_pipeline/test_flows/test_sync_table_with_pandas.py

@@ -81,6 +81,7 @@ def test_sync_table_with_pandas(
     )
 
     assert len(df) > 0
+    breakpoint()
 
     client.command(
         ("DROP TABLE " "{database:Identifier}.{table:Identifier}"),

infra/testing/docker-compose-test-data.yml

@@ -45,6 +45,7 @@ services:
     volumes:
       - ../../forklift/tests/test_data/external/monitorfish/backend/src/main/resources/db/migration:/flyway/sql/schema
       - ../../forklift/tests/test_data/external/monitorfish/datascience/tests/test_data/remote_database/:/flyway/sql/test_data
+      - ../../forklift/tests/test_data/monitorfish/remote_database/:/flyway/sql/extra_test_data
     depends_on:
       monitorfish-db:
         condition: service_healthy