Merge pull request #39 from govlt/setup-parcels

Setup parcels
govlt · Oct 7, 2024 · 5a29048 · 5a29048
2 parents 30d4d1d + 877662b
commit 5a29048
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 18 deletions.
diff --git a/.github/workflows/check-for-updates.yml b/.github/workflows/check-for-updates.yml
@@ -17,10 +17,13 @@ jobs:
     outputs:
       diff: ${{ steps.comparison.outputs.diff }}
     steps:
+      - name: Install csvkit
+        run: apt-get update && apt-get install -y csvkit
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Crete GeoPackage
+      - name: Create GeoPackage
         run: chmod +x create-geopackage.sh && ./create-geopackage.sh
 
       - name: Upload data-source-checksums.txt artifact

diff --git a/.github/workflows/publish-pmtiles.yml b/.github/workflows/publish-pmtiles.yml
@@ -20,10 +20,13 @@ jobs:
     permissions:
       contents: read
     steps:
+      - name: Install csvkit
+        run: apt-get update && apt-get install -y csvkit
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Crete GeoPackage
+      - name: Create GeoPackage
         run: chmod +x create-geopackage.sh && ./create-geopackage.sh
 
       - name: Upload GeoPackage artifact
@@ -143,6 +146,7 @@ jobs:
             pmtiles/elderships.pmtiles
             pmtiles/residential-areas.pmtiles
             pmtiles/streets.pmtiles
+            pmtiles/parcels.pmtiles
             data-source-checksums.txt
 
   publish-page:
@@ -164,10 +168,9 @@ jobs:
           merge-multiple: true
 
       # Cloudflare Pages only supports files up to 25 MiB in size
-      # Temporary delete this file
-      - name: Delete residential-areas.pmtiles
-        run: rm output/pmtiles/residential-areas.pmtiles
-
+      # Temporary delete these file
+      - name: Delete files over 25 MiB
+        run: rm output/pmtiles/residential-areas.pmtiles output/pmtiles/parcels.pmtiles
 
       - name: Upload to Cloudflare pages
         uses: cloudflare/wrangler-action@v3

diff --git a/README.md b/README.md
@@ -20,35 +20,42 @@ documentation.
 Use hosted versions of boundaries on your website, with global low latency and high SLA via Cloudflare Pages.
 
 | Type                  | Lithuanian Translation    | Demo                                                                                                                            | PMTiles Archive URL                                                                       |
-|-----------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------|
+| --------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- |
 | **Counties**          | Apskričių ribos           | [Demo](https://pmtiles.io/?url=https%3A%2F%2Fboundaries.startupgov.lt%2Fpmtiles%2Fcounties.pmtiles#map=6.95/55.191/22.92)       | [counties.pmtiles](https://boundaries.startupgov.lt/pmtiles/counties.pmtiles)             |
 | **Municipalities**    | Savivaldybių ribos        | [Demo](https://pmtiles.io/?url=https%3A%2F%2Fboundaries.startupgov.lt%2Fpmtiles%2Fmunicipalities.pmtiles#map=6.95/55.191/22.92) | [municipalities.pmtiles](https://boundaries.startupgov.lt/pmtiles/municipalities.pmtiles) |
 | **Elderships**        | Seniūnijų ribos           | [Demo](https://pmtiles.io/?url=https%3A%2F%2Fboundaries.startupgov.lt%2Fpmtiles%2Felderships.pmtiles#map=6.95/55.191/22.92)     | [elderships.pmtiles](https://boundaries.startupgov.lt/pmtiles/elderships.pmtiles)         |
 | **Residential Areas** | Gyvenamųjų vietovių ribos | -                                                                                                                               | Not Available                                                                             |
 | **Streets**           | Gatvių ribos              | [Demo](https://pmtiles.io/?url=https%3A%2F%2Fboundaries.startupgov.lt%2Fpmtiles%2Fstreets.pmtiles#map=11/54.6828/25.2686)       | [streets.pmtiles](https://boundaries.startupgov.lt/pmtiles/streets.pmtiles)               |
+| **Parcels**           | Žemės sklypų ribos        | -                                                                                                                               | Not Available                                                                             |
 
 ### Self-Hosting
 
 Download the latest PMTiles archives and host them on your own server.
 
-| Type                  | Lithuanian Translation    | PMTiles Archive URL                                                                                                          |
-|-----------------------|---------------------------|------------------------------------------------------------------------------------------------------------------------------|
-| **Counties**          | Apskričių ribos           | [counties.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/counties.pmtiles)                   |
-| **Municipalities**    | Savivaldybių ribos        | [municipalities.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/municipalities.pmtiles)       |
-| **Elderships**        | Seniūnijų ribos           | [elderships.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/elderships.pmtiles)               |
-| **Residential Areas** | Gyvenamųjų vietovių ribos | [residential-areas.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/residential-areas.pmtiles) |
-| **Streets**           | Gatvių ribos              | [streets.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/streets.pmtiles)                     |
+| Type                  | Lithuanian Translation    | PMTiles Archive URL                                                                                                                                      |
+| --------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Counties**          | Apskričių ribos           | [counties.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/counties.pmtiles)                                               |
+| **Municipalities**    | Savivaldybių ribos        | [municipalities.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/municipalities.pmtiles)                                   |
+| **Elderships**        | Seniūnijų ribos           | [elderships.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/elderships.pmtiles)                                           |
+| **Residential Areas** | Gyvenamųjų vietovių ribos | [residential-areas.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/residential-areas.pmtiles)                             |
+| **Streets**           | Gatvių ribos              | [streets.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/streets.pmtiles)                                                 |
+| **Parcels**           | Žemės sklypų ribos        | [parcels.pmtiles](https://github.com/govlt/national-boundaries/releases/latest/download/parcels.pmtiles)<br/>(**Important:** visible from zoom level 14) |
 
 ## Architecture
 
 ```mermaid
 flowchart TD
-    ar["State Enterprise Centre of Registers\n<a href='https://www.registrucentras.lt/p/1187'>Address Registry raw data</a>"]
-    --> transform["<a href='https://github.com/govlt/national-boundaries/blob/main/create-geopackage.sh'>Create GeoPackage</a>"]
-    --> |"<a href='https://github.com/govlt/national-boundaries/releases/latest/download/boundaries-4326.gpkg'>boundaries-4326.gpkg</a>"|github-releases
+    rc["<a href='https://www.registrucentras.lt'>State Enterprise Centre of Registers</a>"]
+    rc-->ar["<a href='https://www.registrucentras.lt/p/1187'>Address Registry raw data</a>"]
+    rc-->pr["<a href='https://www.registrucentras.lt/p/1092'>Parcels raw data</a>"]
 
+    transform["<a href='https://github.com/govlt/national-boundaries/blob/main/create-geopackage.sh'>Create GeoPackage</a>"]-->|"<a href='https://github.com/govlt/national-boundaries/releases/latest/download/boundaries-4326.gpkg'>boundaries-4326.gpkg</a>"|github-releases    
+    
     github-releases["<a href='https://github.com/govlt/national-boundaries/releases'>GitHub Releases</a>"]--> cloudflare-pages["Cloudflare Pages"]
 
+    ar-->transform
+    pr-->transform
+    
     cloudflare-pages-->pages-counties["<a href='https://boundaries.startupgov.lt/pmtiles/counties.pmtiles'>counties.pmtiles</a>"]
     cloudflare-pages-->pages-municipalities["<a href='https://boundaries.startupgov.lt/pmtiles/municipalities.pmtiles'>municipalities.pmtiles</a>"]
     cloudflare-pages-->pages-elderships["<a href='https://boundaries.startupgov.lt/pmtiles/elderships.pmtiles'>elderships.pmtiles</a>"]
@@ -91,4 +98,4 @@ English. For full details, check out the [create-geopackage.sh](./create-geopack
 ## Contributing
 
 We welcome contributions! For details, see
-our [contribution guidelines](https://github.com/govlt/.github/blob/main/CONTRIBUTING.md).
+our [contribution guidelines](https://github.com/govlt/.github/blob/main/CONTRIBUTING.md).
diff --git a/create-geopackage.sh b/create-geopackage.sh
@@ -58,6 +58,20 @@ download_data_source_and_md5 streets.json "https://www.registrucentras.lt/aduome
 ogr2ogr -append boundaries-4326.gpkg data-sources/streets.json -lco FID=feature_id -xyRes 0.000001 -t_srs EPSG:4326 \
   -sql "SELECT FID AS feature_id, GAT_KODAS AS code, GAT_PAV as name, GAT_PAV_PI AS full_name, GAT_ILGIS as length_m, GYV_KODAS AS residential_area_code, GTV_R AS created_at FROM streets"
 
+echo "Importing parcels data for each municipality"
+curl -sf "https://www.registrucentras.lt/aduomenys/?byla=adr_savivaldybes.csv" | csvcut -d "|" -c "SAV_KODAS" | tail -n +2 | while read -r code; do
+  echo "Converting https://www.registrucentras.lt/aduomenys/?byla=gis_pub_parcels_$code.zip"
+  curl -f -L --max-redirs 5 --retry 3 -o "data-sources/parcels-$code.zip" "https://www.registrucentras.lt/aduomenys/?byla=gis_pub_parcels_$code.zip"
+  calculate_md5 "data-sources/parcels-$code.zip" >> data-sources/data-source-checksums.txt
+  unzip data-sources/parcels-$code.zip -d data-sources
+
+  ogr2ogr -append -f GPKG data-sources/parcels.gpkg "data-sources/gis_pub_parcels_$code.json" -nln polygons
+done
+
+echo "Finishing parcels data import into GeoPackage"
+ogr2ogr -append boundaries-4326.gpkg data-sources/parcels.gpkg -nln parcels -xyRes 0.000001 -t_srs EPSG:4326 \
+  -sql "SELECT polygons.unikalus_nr AS unique_number, CAST(polygons.pask_tipas AS integer(8)) AS purpose_id, CAST(polygons.osta_statusas AS integer(8)) AS status_id, polygons.geom, polygons.kadastro_nr as cadastral_number, CAST(polygons.sav_kodas AS integer(8)) AS municipality_code, CAST(polygons.seniunijos_kodas AS integer(8)) AS eldership_code, CAST(polygons.skl_plotas AS FLOAT) as area_ha FROM polygons"
+
 echo "Finalizing GeoPackage"
 ogrinfo boundaries-4326.gpkg -sql "VACUUM"
 

diff --git a/preview/tileserver-config.json b/preview/tileserver-config.json
@@ -14,6 +14,9 @@
     },
     "streets": {
       "pmtiles": "/opt/data/streets.pmtiles"
+    },
+    "parcels": {
+      "pmtiles": "/opt/data/parcels.pmtiles"
     }
   }
 }
diff --git a/src/main/java/lt/startupgov/boundaries/Main.java b/src/main/java/lt/startupgov/boundaries/Main.java
@@ -43,6 +43,11 @@ private record LayerConfiguration(
                     "streets.pmtiles",
                     new Streets()
             ),
+            new LayerConfiguration(
+                    "Parcels of Lithuania",
+                    "parcels.pmtiles",
+                    new Parcels()
+            ),
     };
 
     public static void main(String[] args) {

diff --git a/src/main/java/lt/startupgov/boundaries/constants/Layers.java b/src/main/java/lt/startupgov/boundaries/constants/Layers.java
@@ -6,6 +6,7 @@ public class Layers {
     public static final String ELDERSHIPS = "elderships";
     public static final String RESIDENTIAL_AREAS = "residential_areas";
     public static final String STREETS = "streets";
+    public static final String PARCELS = "parcels";
 
     private Layers() {
     }

diff --git a/src/main/java/lt/startupgov/boundaries/layers/Parcels.java b/src/main/java/lt/startupgov/boundaries/layers/Parcels.java
@@ -0,0 +1,32 @@
+package lt.startupgov.boundaries.layers;
+
+import com.onthegomap.planetiler.FeatureCollector;
+import com.onthegomap.planetiler.reader.SourceFeature;
+import lt.startupgov.boundaries.constants.Layers;
+import lt.startupgov.boundaries.constants.Source;
+
+public class Parcels implements Layer {
+
+    @Override
+    public void processFeature(SourceFeature sf, FeatureCollector features) {
+        if (sf.getSource().equals(Source.BOUNDARIES) && sf.getSourceLayer().equals(Layers.PARCELS) && sf.canBePolygon()) {
+            // We do not have feature ID right now
+            var purpose_id = sf.getLong("purpose_id");
+            var status_id = sf.getLong("status_id");
+            var eldership_code = sf.getLong("eldership_code");
+            var municipality_code = sf.getLong("municipality_code");
+
+            features.polygon(Layers.PARCELS)
+                    .setBufferPixels(4)
+                    .setMinPixelSizeAtAllZooms(0)
+                    .setMinZoom(14)
+                    .setAttr("unique_number", sf.getLong("unique_number"))
+                    .setAttr("cadastral_number", sf.getString("cadastral_number"))
+                    .setAttr("status_id", status_id != 0 ? status_id : null)
+                    .setAttr("purpose_id", purpose_id != 0 ? purpose_id : null)
+                    .setAttr("area_ha", sf.getString("area_ha"))
+                    .setAttr("municipality_code", municipality_code != 0 ? municipality_code : null)
+                    .setAttr("eldership_code", eldership_code != 0 ? eldership_code : null);
+        }
+    }
+}