diff --git a/.kodiak.toml b/.kodiak.toml deleted file mode 100644 index 1350d9ee0..000000000 --- a/.kodiak.toml +++ /dev/null @@ -1,13 +0,0 @@ -version = 1 - -[approve] -auto_approve_usernames = ["dependabot"] - -[merge] -method = "squash" -automerge_label = "automerge 🚀" - -[merge.message] -title = "pull_request_title" -include_coauthors = true -include_pull_request_url = true \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e903091a..5e0e20246 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,58 @@ We would love for you to contribute to basemaps and help make it even better than it is today! As a contributor, here are the guidelines we would like you to follow: -## Monorepo +## Building + +This repository requires [NodeJs](https://nodejs.org/en/) > 16 & [Yarn](https://yarnpkg.com/en/) + +Use [n](https://github.com/tj/n) or [fnm](https://github.com/Schniz/fnm) to manage nodeJs versions + +```bash +# Download the latest nodejs & yarn +n latest +npm install -g yarn + +# Install node deps +yarn + +# Build everything into /build +yarn run build + +# Run the unit tests +yarn run test +``` + +For a fresh rebuild + +```bash +yarn run clean +``` + +## Packages this repository is a monorepo for everything related to basemaps -for explanations for each component look for the README.md inside each package. + +- [@basemaps/test](packages/__tests__/) - Testing utilities and assets +- [@basemaps/infra](packages/_infra/) - Infrastructure code using [AWS CDK](https://github.com/aws/aws-cdk) +- [@basemaps/attribution](packages/attribution/) - Calculate the attribution for given map location +- [@basemaps/bathymetry](packages/bathymetry/) - Convert bathymetry from [GEBCO](https://www.gebco.net/) into colorized HillShade geotiff. +- [@basemaps/cli](packages/cli/) - cli that using for CICD process +- [@basemaps/cogify](packages/cogify/) - CLI to re-tile imagery into a Cloud Optimised Geotiffs (COG) +- [@basemaps/config](packages/config/) - Configurations for Basemaps system +- [@basemaps/geo](packages/geo/) - Utility to work with QuadKeys, Tiles and Projections. +- [@basemaps/lambda-analytics](packages/lambda-analytics/) - Generate analytics from CloudFront distribution statistics +- [@basemaps/lambda-tiler](packages/lambda-tiler/) - Lambda server for WMTS/XYZ map generation +- [@basemaps/landing](packages/landing/) - The landing page for Basemaps +- [@basemaps/server](packages/server/) - cli for WMTS/XYZ Tile server +- [@basemaps/shared](packages/shared/) - Shared Utilities for other Basemaps packages +- [@basemaps/smoke](packages/smoke/) - Smoke tests +- [@basemaps/sprites](packages/sprites/) - sprite sheet generation +- [@basemaps/tiler](packages/tiler/) - Compose CogGeoTiffs for xyz tile server +- [@basemaps/tiler-sharp](packages/tiler-sharp/) - Generate tiles by using [sharp](https://github.com/lovell/sharp) and [libvips](https://github.com/libvips/libvips) +- [@linzjs/docker-command](packages/linzjs-docker-command/) - Utilities for running commands inside Docker +- [@linzjs/geojson](packages/linzjs-geojson/) - Utility for working with GeoJSON +- [@linzjs/metrics](packages/linzjs-metrics/) - Simple timing metric tracker for NodeJS + ## Commit message diff --git a/README.md b/README.md index abf5b199c..9c2bde78f 100644 --- a/README.md +++ b/README.md @@ -2,77 +2,23 @@ [![Build Status](https://github.com/linz/basemaps/workflows/Build/badge.svg)](https://github.com/linz/basemaps/actions) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/linz/basemaps/blob/master/LICENSE) -[![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/linz/basemaps.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/linz/basemaps/context:javascript) -A digital basemap provides a consistent background detail necessary to orient location and add to aesthetic appeal. Basemaps can be made up of streets, parcels, boundaries (country, regional, and city boundaries), shaded relief of a digital elevation model, waterways, hydrography, aerial and satellite imagery. Basemaps can be used as desktop, website or mobile phone application components, or as a 3rd party layers within a GIS or desktop mapping application. - -## Packages: - -- [@basemaps/test](packages/__tests__/) - Testing utilities and assets -- [@basemaps/infra](packages/_infra/) - Infrastructure code using [AWS CDK](https://github.com/aws/aws-cdk) -- [@basemaps/attribution](packages/attribution/) - Calculate the attribution for given map location -- [@basemaps/bathymetry](packages/bathymetry/) - Convert bathymetry from [GEBCO](https://www.gebco.net/) into colorized HillShade geotiff. -- [@basemaps/cli](packages/cli/) - cli that using for CICD process -- [@basemaps/cogify](packages/cogify/) - CLI to re-tile imagery into a Cloud Optimised Geotiffs (COG) -- [@basemaps/config](packages/config/) - Configurations for Basemaps system -- [@basemaps/geo](packages/geo/) - Utility to work with QuadKeys, Tiles and Projections. -- [@basemaps/lambda-analytics](packages/lambda-analytics/) - Generate analytics from CloudFront distribution statistics -- [@basemaps/lambda-tiler](packages/lambda-tiler/) - Lambda server for WMTS/XYZ map generation -- [@basemaps/landing](packages/landing/) - The landing page for Basemaps -- [@basemaps/server](packages/server/) - cli for WMTS/XYZ Tile server -- [@basemaps/shared](packages/shared/) - Shared Utilities for other Basemaps packages -- [@basemaps/smoke](packages/smoke/) - Smoke tests -- [@basemaps/sprites](packages/sprites/) - sprite sheet generation -- [@basemaps/tiler](packages/tiler/) - Compose CogGeoTiffs for xyz tile server -- [@basemaps/tiler-sharp](packages/tiler-sharp/) - Generate tiles by using [sharp](https://github.com/lovell/sharp) and [libvips](https://github.com/libvips/libvips) -- [@linzjs/docker-command](packages/linzjs-docker-command/) - Utilities for running commands inside Docker -- [@linzjs/geojson](packages/linzjs-geojson/) - Utility for working with GeoJSON -- [@linzjs/metrics](packages/linzjs-metrics/) - Simple timing metric tracker for NodeJS - -## Building - -This repository requires [NodeJs](https://nodejs.org/en/) > 16 & [Yarn](https://yarnpkg.com/en/) - -Use [n](https://github.com/tj/n) or [fnm](https://github.com/Schniz/fnm) to manage nodeJs versions - -```bash -# Download the latest nodejs & yarn -n latest -npm install -g yarn -# Install node deps -yarn - -# Build everything into /build -yarn run build - -# Run the unit tests -yarn run test -``` - -## Example clients +A digital basemap provides a consistent background detail necessary to orient location and add to aesthetic appeal. Basemaps can be made up of streets, parcels, boundaries (country, regional, and city boundaries), shaded relief of a digital elevation model, waterways, hydrography, aerial and satellite imagery. Basemaps can be used as desktop, website or mobile phone application components, or as a 3rd party layers within a GIS or desktop mapping application. -See https://basemaps.linz.govt.nz/examples ## License This system is licensed under the MIT License, except where otherwise specified. See the [LICENSE](https://github.com/linz/basemaps/blob/master/LICENSE) file for more details. -## Deployment +## Development -Deployments of Basemaps are managed with github actions. +For background information about the basemaps service and its development see [Documentation](./docs/README.md) -To trigger a deployment, make sure your branch is up to date and run the version bump script [./scripts/version.bump.sh](./scripts/version.bump.sh) -This script will create a `release:` commit and branch, please review the commit then create a pull request from it. - -Once the release pull request is merged the CI system will deploy the released version into dev then into production. - -## Deployment Rollback +## Building -when a Deployment Breaks Badly, don't try to fix that on the fly, this risk of introducing more errors and downtime. The fastest way is to rollback to previous release immediately and fix the problem before next release. +For building and contributing guide see [Contributing](./CONTRIBUTING.md) -As Basemaps deployments are managed with github actions, every release will bundle the release packages and deployment in the github action automatically. So, it is very simple for use to roll back to previous release as all the previous deployment are remained in the github actions history. Please use the following steps to trigger a deployment roll when needed. +## Changes -- On the main page of the repository, above the file list, click :clock4: commits -- To navigate to the previous release commit, click the tick mark to see all the previous github actions for this commit. -- Select `Build / deploy-prod (push)` workflow and rerun the it to roll back to previous release. +For a history of basemaps changes see the [Changelog](./CHANGELOG.md) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index e01c2931e..8009d001d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,73 @@ -# Overview +# LINZ Basemaps -## What is LINZ Basemaps? +[LINZ Basemaps](https://basemaps.linz.govt.nz) is a collection of tools to create and serve vector and raster basemaps using open source and open standards. It is designed to be light weight, cost efficient and fast. -A digital basemap provides a consistent background detail necessary to orient location and add to aesthetic appeal. Basemaps can be made up of streets, parcels, boundaries (country, regional, and city boundaries), shaded relief of a digital elevation model, waterways, hydrography, aerial and satellite imagery. Basemaps can be used as desktop, website or mobile phone application components, or as a 3rd party layers within a GIS or desktop mapping application. +Basemaps currently supports both [Imagery](#aerial--satellite-imagery) and [Vector data](#vector-data) + +## Background + +Land Information New Zealand has a large archive of aerial and satellite imagery freely available from [LINZ Data Service](https://data.linz.govt.nz) and the [Registry of Open Data on AWS](https://registry.opendata.aws/nz-imagery/). This imagery was a primary driver for the creation of the LINZ Basemaps product and service. For more information on how LINZ acquires, ingests and publishes aerial and satellite imagery see [linz/imagery](https://github.com/linz/imagery). + +While linz/basemaps is public, its primary reason is to [work in the open](https://www.digital.govt.nz/standards-and-guidance/digital-service-design-standard/principles/work-in-the-open/), so that we can share knowledge and foster collaboration with peers across the world. While the LINZ Basemaps product and services can be customised and deployed we are primarily focused on our own use cases of sharing New Zealand's Aerial Imagery archive and Topographic vector data. + +## Basemaps Service + +![Basemaps](./static/basemaps-service.png) The core of the LINZ Basemaps Service consists of + +- AWS S3 - COG Storage location +- AWS Lambda function - Tile Server - [@basemaps/lambda-tiler](../packages/lambda-tiler/README.md) +- AWS Cloudfront - Content distribution and tile cache +- [linz/basemaps-config](https://github.com/linz/basemaps) - Basemaps configuration + +### Imagery Tile Requests + +A tile request flows a number of basemaps packages: + +Given a request `/v1/tiles/aerial/WebMercatorQuad/6/1/40.webp`, +Breaking down this request url `lambda-tiler` needs to serve a `webp` image that for the tile `{z: 6, x: 1, y: 40}` from the `WebMercatorQuad` tile matrix from the tile set named `aerial` in the default configuration. + +Which will then utlize the following packages + +- [@basemaps/config](../packages/config/README.md) - Load configuration ([`?config`](./configuration.md)) +- [@basemaps/lambda-tiler](../packages/lambda-tiler/README.md) Convert tile XYZ into output coordinates bounding box +- [@basemaps/lambda-tiler](../packages/lambda-tiler/README.md) Determine which tiffs intersect the output coordinates +- [@basemaps/tiler](../packages/tiler/README.md) Determine which parts of the tiffs need to be loaded and resampled +- [@basemaps/tiler-sharp](../packages/tiler-sharp/README.md) - Load the bytes from the tiff and render it as a output image. + +Imagery is stored as cloud optimised geotiffs by survey then layered together dynamically based on a [configured priority order](./configuration.md). + +[@basemaps/tiler](../packages/tiler/README.md) takes the collection of tiffs and determines how resample (resize/crop) and paint them onto the output tile. + +![Tiff Operations](./static/tile-resize.png) + +[@basemaps/tiler-sharp](../packages/tiler-sharp/README.md) uses [sharp/libvips](https://github.com/lovell/sharp) to take the output of [@basemaps/tiler](../packages/tiler/README.md) then load the raw tiff bytes and render them into output image. + +### Vector tile requests + +Vector tiles are pre-rendered as [Mapbox Vector Tiles](https://docs.mapbox.com/data/tilesets/guides/vector-tiles-introduction/) and served directly from a [Cloud Optimised Tar](https://github.com/linz/cotar). Since there is no merging or dynamic configuration the tile server will just request the specified tile directly from the tar file. + +### Additional Processing + +#### Aerial & Satellite Imagery Processing + +To efficiently serve aerial imagery to huge number of consumers, it should be optimised into a format that makes it easy to serve. All of LINZ's imagery is stored as [Cloud Optimised GeoTIFFs (COGs)](https://www.cogeo.org/), and to ensure they are served as efficiently as possible some additional processing steps are generally pre-applied to the imagery. + +- Reprojection: reprojecting the imagery into common consumption formats (EPSG:3857 and EPSG:2193) will greatly reduce the amount of load and complexity of the basemaps service service +- Tile Alignment: By aligning the COGs to the output XYZ tile grid +- Additional overviews: It is hard to fetch data from 1,000s of COGs to create a single tile, so we create additional overviews up to z0. + +These additional processing steps are part of the basemaps [import process](#TODO) and are controlled by [@basemaps/cogify](../packages/cogify/README.md) + +The additional steps are optional but do **greatly improve** the tile serving performance of the service. + +#### Vector Data + +Vector data is processed from the LINZ topographic vector datasets using tippecanoe, this process is still a work in progress and will. + +## Development + +Deployments are handled by Github Actions see [Deployments](./deployment.md) for more information. + +### Diagrams + +All diagrams across this repository are created with [excalidraw](https://excalidraw.com/) and embed the excalidraw configuration inside them, so can be re-edited on https://excalidraw.com/ diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..bb454420d --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,48 @@ +## Configuration + +Basemaps has two main components to its configuration + +- Imagery - List of tiffs that make up a imagery set +- TileSet - List of imagery or vector layers that make up a output layer. + +The configuration can be stored as individual AWS DynamoDB objects or can be stored as a bundled JSON object. + +The configuration is generally generated directly from the source tiffs and associated STAC metadata using the [@basemaps/cli](../packages/cli/README.md) import process, with LINZ's specific configuration being stored in [linz/basemaps-config](https://github.com/linz/basemaps-config). + +This allows the configuration objects to be somewhat short, below is a snippet from the [aerial](https://github.com/linz/basemaps-config/blob/master/config/tileset/aerial.json) tile set configuration which layers 100+ aerial imagery layers together. + +```json5 +{ + // Raster or Vector layer + "type": "raster", + // all id's are prefixed by type eg 'ts_' is TileSet + "id": "ts_aerial", + // Human friendly name + "title": "Aerial Imagery Basemap", + "category": "Basemaps", + // If there is no data create a background color as hex RGBA + "background": "dce9edff", + "layers": [ + { + // Source location for EPSG:2193 Imagery + "2193": "s3://linz-basemaps/2193/gebco_2020_Nztm2000Quad_305-75m/01F1BFJN8R8P7BXN3XTHC5MT5G", + // Source location for EPSG:3857 Imagery + "3857": "s3://linz-basemaps/3857/gebco_2020_305-75m/01EDMTM3P563P06TWYQAZRA9F6", + // url/slug friendly name of the imagery set + "name": "gebco-2020-305.75m", + // At what level should the imagery be turned off + "maxZoom": 15, + // Human friendly name for the imagery + "title": "GEBCO Gridded Bathymetry (2020)", + "category": "Bathymetry" + }, + ... + ] +} +``` + +When this configuration is imported it will list all of the source locations looking for `collection.json` and `*.tiff|*.tif` files. + +### Dynamic configuration + +Configuration can be stored as single JSON object in s3 and then passed directly to the basemaps service with `?config=s3://linz-basemaps/config/config-latest.json.gz` this allows basemaps to preview a configuration before being deployed. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 000000000..affb9e45a --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,22 @@ +# Deployment + +Deployments of Basemaps are managed with github actions. + +To trigger a deployment, make sure your branch is up to date and run the version bump script [../scripts/version.bump.sh](../scripts/version.bump.sh) +This script will create a `release:` commit and branch, please review the commit then create a pull request from it. + +Once the release pull request is merged the CI system will deploy the released version into dev then into production. + +## Deployment Rollback + +If a deployment breaks production, don't try to fix it on the fly, this risks introducing more errors and downtime. The fastest way to ensure production is stable again is to roll back to the previous release immediately and then focus on fixing the problem before the next release. + +As Basemaps deployments are managed with GitHub Actions, every release will bundle the release packages and deployment in the GitHub Action run automatically. So, it is very simple for us to roll back to a previous release as all the previous deployments remain in history. Please use the following steps to trigger a roll back when needed. + +- Open the Basemaps Deployments page - https://github.com/linz/basemaps/deployments +- Choose the environment that needs to be rolled back +- Open the workflow run for an older release + +![Workflow runs for older releases](./static/workflow-run.png | width=400) + +- Select the `Build / deploy-prod (push)` workflow and re-run it to roll back to a previous release. diff --git a/docs/operator-guide/cog-quality.md b/docs/operator-guide/cog-quality.md index e739c0cbe..43841be5f 100644 --- a/docs/operator-guide/cog-quality.md +++ b/docs/operator-guide/cog-quality.md @@ -30,7 +30,6 @@ gdal_translate \ done ``` - ## Overviews When creating COGs using the different resampling methods the output quality differs a lot. @@ -39,13 +38,14 @@ Here are some overviews created using some of the different resampling methods: ![Resampling Quality](./static/quality__resampling-overview.webp) -### Lanczos +### Lanczos Provides quite a nice and crisp output ![Lanczos Overview](./static/quality__i6.lanczos.webp) ### Cubic + Creates weird artifacts around the edges ![Cubic Overview](./static/quality__i6.cubic.webp) @@ -56,7 +56,6 @@ looks blury ![Bilinear Overview](./static/quality__i6.bilinear.webp) - Using lanczos looks best for our overview creation. ## Reprojection resampling @@ -77,12 +76,10 @@ Creates a nice image ![Bilinear](./static/quality__005_006_0_bilinear.webp) - For our aerial imagery reprojection seems to be best done as bilinear, where as Overview creation seems to be best done as lanczos. - Using GDAL 3.2.x+ this can be configured with as it was fixed in [osgeo/gdal#2671](https://github.com/OSGeo/gdal/issues/2671) ```bash gdal_translate -co WARP_RESAMPLING=bilinear -co OVERVIEW_RESAMPLING=lanczos -``` \ No newline at end of file +``` diff --git a/docs/operator-guide/empty-tiles.md b/docs/operator-guide/empty-tiles.md index 4548dbf16..2d3c36799 100644 --- a/docs/operator-guide/empty-tiles.md +++ b/docs/operator-guide/empty-tiles.md @@ -62,7 +62,7 @@ Pros: - ✔️ Supported in ArcGIS Pro and QGIS - ✔️ Supported by Leaflet and OpenLayers -- ✔️ Supported by [HTTP RFC](https://www.rfc-editor.org/rfc/rfc9110.html#name-200-ok): "server ought to return 204 No content" +- ✔️ Supported by [HTTP RFC](https://www.rfc-editor.org/rfc/rfc9110.html#name-200-ok): "server ought to return 204 No content" - ✔️ Recommended option from OGC Tile API Cons: diff --git a/docs/operator-guide/relief-shade.md b/docs/operator-guide/relief-shade.md index ea572dea4..578fae3cf 100644 --- a/docs/operator-guide/relief-shade.md +++ b/docs/operator-guide/relief-shade.md @@ -5,8 +5,8 @@ - 8M DEM of NZ - [lds:51768](https://data.linz.govt.nz/layer/51768-nz-8m-digital-elevation-model-2012/) - GDAL - Create a VRT of all the imagery + ```bash gdalbuildvrt dem/*.tif dem.vrt ``` @@ -37,7 +37,6 @@ Base Image: gdaldem hillshade: ![Aerial imagery](./static/relief__base.webp) - Using default color ramp applys whites ![Lighten](./static/relief__lighten.webp) @@ -51,6 +50,6 @@ nv 0 0 0 0 255 0 0 0 0 ``` -`nv` - No value or NO_DATA set full alpha +`nv` - No value or NO_DATA set full alpha -![Lighten](./static/relief__darken.webp) \ No newline at end of file +![Lighten](./static/relief__darken.webp) diff --git a/docs/operator-guide/texture-shade.md b/docs/operator-guide/texture-shade.md index dd109a6cc..dc387f2ad 100644 --- a/docs/operator-guide/texture-shade.md +++ b/docs/operator-guide/texture-shade.md @@ -4,7 +4,6 @@ See http://www.textureshading.com/Home.html - ## Process ```bash @@ -14,7 +13,7 @@ See http://www.textureshading.com/Home.html # # This process requires a huge amount of memory, for the 8M geographx DEM it requires around 100GB of free memory # This process has been tested on a AWS R5D with 32VCPUs and 256GB of ram, starting with a base AWS centos image -# +# ################# # Install required software sudo yum groupinstall 'Development Tools' @@ -24,7 +23,7 @@ sudo yum install tmux git sudo systemctl start docker sudo docker ps -# Helper function to run docker +# Helper function to run docker function gdal_docker() { sudo docker run --rm -it -v $PWD:$PWD --workdir $PWD osgeo/gdal:ubuntu-small-3.3.3 "$@" } @@ -70,7 +69,7 @@ gdal_docker gdalinfo Geographx-NZ-DEM-FLT.flt sudo chown ssm-user:ssm-user n1 -R # Texture shading takes a while run inside of tmux so the session can be resumed with `tmux attach` -tmux +tmux ../texture-shade/bin/texture 1 Geographx-NZ-DEM-FLT.flt Geographx-NZ-DEM-FLT.detail_1.flt ../texture-shade/bin/texture_image +1.5 Geographx-NZ-DEM-FLT.detail_1.flt Geographx-NZ-DEM-FLT.detail_1.contrast_1.5.tif # texture-shade only allows ".tif" @@ -84,11 +83,10 @@ gdal_docker gdal_translate -of Gtiff -co COMPRESS=lzw -co BIGTIFF=yes -co NUM_TH aws s3 cp Geographx-NZ-DEM-FLT.detail_1.contrast_1.5.lzw.tiff s3://linz-basemaps-source/Geographx-NZ-DEM-FLT/ ``` - Once complete this gives a uint16 tiff that has values between 0-65,536 to make this image usage a color ramp is applied to create a semi transparent image - At 0 create a 25% opacity black, `rgba(0,0,0,0.25)` then slowly ramp to 0% opacity at 32,768 + ``` nv 0 0 0 0 0 0 0 0 63 @@ -97,7 +95,6 @@ nv 0 0 0 0 Using gdal apply the color-relief - ```bash gdaldem color-relief \ -co COMPRESS=lzw \ @@ -107,4 +104,4 @@ gdaldem color-relief \ -co BIGTIFF=yes \ -co ALPHA=yes -alpha \ Geographx-NZ-DEM-FLT.detail_1.contrast_1.5.lzw.tiff color.ramp Geographx-NZ-DEM-FLT.detail_1.contrast_1.5.lzw.ramp.tiff -``` \ No newline at end of file +``` diff --git a/docs/static/basemaps-service.png b/docs/static/basemaps-service.png new file mode 100644 index 000000000..c63ab08cd Binary files /dev/null and b/docs/static/basemaps-service.png differ diff --git a/docs/static/tile-resize.png b/docs/static/tile-resize.png new file mode 100644 index 000000000..070d41286 Binary files /dev/null and b/docs/static/tile-resize.png differ diff --git a/docs/static/workflow-run.png b/docs/static/workflow-run.png new file mode 100644 index 000000000..220c2133a Binary files /dev/null and b/docs/static/workflow-run.png differ