diff --git a/.circleci/config.yml b/.circleci/config.yml index 32e5a1e18..e9d3844bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ _defaults: &defaults environment: TERM: dumb docker: - - image: circleci/openjdk:8-jdk + - image: s22s/rasterframes-circleci:latest _setenv: &setenv name: set CloudRepo credentials @@ -19,81 +19,111 @@ _delenv: &unsetenv _restore_cache: &restore_cache keys: - - v2-dependencies-{{ checksum "build.sbt" }} - - v2-dependencies- + - v3-dependencies-{{ checksum "build.sbt" }} + - v3-dependencies- _save_cache: &save_cache - key: v2-dependencies--{{ checksum "build.sbt" }} + key: v3-dependencies--{{ checksum "build.sbt" }} paths: + - ~/.cache/coursier - ~/.ivy2/cache - ~/.sbt - - ~/.rf_cache + - ~/.local jobs: - staticAnalysis: + test: <<: *defaults - + resource_class: large steps: - checkout - run: *setenv - restore_cache: <<: *restore_cache - - run: cat /dev/null | sbt dependencyCheck - - run: cat /dev/null | sbt --debug dumpLicenseReport + - run: ulimit -c unlimited -S + - run: cat /dev/null | sbt -batch core/test datasource/test experimental/test pyrasterframes/test + - run: + command: | + mkdir -p /tmp/core_dumps + cp core.* *.hs /tmp/core_dumps 2> /dev/null || true + when: on_fail - - run: *unsetenv + - store_artifacts: + path: /tmp/core_dumps + - run: *unsetenv - save_cache: <<: *save_cache - - store_artifacts: - path: datasource/target/scala-2.11/dependency-check-report.html - destination: dependency-check-report-datasource.html - - store_artifacts: - path: experimental/target/scala-2.11/dependency-check-report.html - destination: dependency-check-report-experimental.html - - store_artifacts: - path: core/target/scala-2.11/dependency-check-report.html - destination: dependency-check-report-core.html - - store_artifacts: - path: pyrasterframes/target/scala-2.11/dependency-check-report.html - destination: dependency-check-report-pyrasterframes.html - test: + docs: <<: *defaults - resource_class: large + resource_class: xlarge steps: - checkout - run: *setenv + - restore_cache: <<: *restore_cache - - run: sudo apt-get install python-pip pandoc && pip install setuptools # required for pyrasterframes testing - - run: cat /dev/null | sbt test + - run: ulimit -c unlimited -S + - run: pip3 install --quiet --user -r pyrasterframes/src/main/python/requirements.txt + - run: + command: cat /dev/null | sbt makeSite + no_output_timeout: 30m + + - run: + command: | + mkdir -p /tmp/core_dumps + cp core.* *.hs /tmp/core_dumps 2> /dev/null || true + when: on_fail + + - store_artifacts: + path: /tmp/core_dumps + + - store_artifacts: + path: docs/target/site + destination: rf-site - run: *unsetenv + - save_cache: <<: *save_cache - publish: + it: <<: *defaults - resource_class: large + resource_class: xlarge steps: - checkout - run: *setenv + - restore_cache: <<: *restore_cache - - run: sudo apt-get install python-pip pandoc && pip install setuptools # required for pyrasterframes testing - - run: cat /dev/null | sbt test - - run: cat /dev/null | sbt publish + - run: ulimit -c unlimited -S + - run: + command: cat /dev/null | sbt it:test + no_output_timeout: 30m + + - run: + command: | + mkdir -p /tmp/core_dumps + cp core.* *.hs /tmp/core_dumps 2> /dev/null || true + when: on_fail + + - store_artifacts: + path: /tmp/core_dumps - run: *unsetenv + - save_cache: <<: *save_cache - it: - <<: *defaults + itWithoutGdal: + working_directory: ~/repo + environment: + TERM: dumb + docker: + - image: circleci/openjdk:8-jdk resource_class: xlarge steps: - checkout @@ -110,6 +140,36 @@ jobs: - save_cache: <<: *save_cache + staticAnalysis: + <<: *defaults + + steps: + - checkout + - run: *setenv + - restore_cache: + <<: *restore_cache + + - run: cat /dev/null | sbt dependencyCheck + - run: cat /dev/null | sbt --debug dumpLicenseReport + + - run: *unsetenv + + - save_cache: + <<: *save_cache + - store_artifacts: + path: datasource/target/scala-2.11/dependency-check-report.html + destination: dependency-check-report-datasource.html + - store_artifacts: + path: experimental/target/scala-2.11/dependency-check-report.html + destination: dependency-check-report-experimental.html + - store_artifacts: + path: core/target/scala-2.11/dependency-check-report.html + destination: dependency-check-report-core.html + - store_artifacts: + path: pyrasterframes/target/scala-2.11/dependency-check-report.html + destination: dependency-check-report-pyrasterframes.html + + workflows: version: 2 all: @@ -119,20 +179,29 @@ workflows: filters: branches: only: - - /astraea\/feature\/.*-its/ - - publish: + - /feature\/.*-its/ + - itWithoutGdal: filters: branches: only: - - astraea/develop - nightlyReleaseAstraea: + - /feature\/.*-its/ + - docs: + filters: + branches: + only: + - /feature\/.*docs.*/ + - /docs\/.*/ + + nightly: triggers: - schedule: cron: "0 8 * * *" filters: branches: only: - - astraea/develop + - develop jobs: - it - - staticAnalysis + - itWithoutGdal + - docs +# - staticAnalysis diff --git a/.gitignore b/.gitignore index ca41e7212..ff43c9712 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ metastore_db tour/jars tour/*.tiff scoverage-report* + +zz-* diff --git a/.scalafmt.conf b/.scalafmt.conf index 4d09e93c7..ca5e10394 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,10 +1,7 @@ maxColumn = 138 continuationIndent.defnSite = 2 -continuationIndent.callSite = 2 -continuationIndent.extendSite = 2 binPack.parentConstructors = true binPack.literalArgumentLists = false -binPack.unsafeCallSite = true newlines.penalizeSingleSelectMultiArgList = false newlines.sometimesBeforeColonInMethodReturnType = false align.openParenCallSite = false @@ -16,5 +13,4 @@ rewriteTokens { } optIn.selfAnnotationNewline = false optIn.breakChainOnFirstMethodDot = true -optIn.configStyleArguments = false -importSelectors = BinPack +importSelectors = BinPack \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index fbe2823fa..12cad75b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,37 +1,38 @@ sudo: false +dist: xenial +language: python -language: scala +python: + - "3.7" cache: directories: - $HOME/.ivy2/cache - $HOME/.sbt/boot - $HOME/.rf_cache + - $HOME/.cache/coursier scala: - 2.11.11 -jdk: - - oraclejdk8 +env: + - COURSIER_VERBOSITY=-1 JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 addons: apt: packages: + - openjdk-8-jdk - pandoc - - python-pip install: - - pip install setuptools - -sbt_args: -no-colors + - pip install rasterio shapely pandas numpy pweave + - wget -O - https://piccolo.link/sbt-1.2.8.tgz | tar xzf - script: - - sbt test - - sbt it:test + - sbt/bin/sbt -java-home $JAVA_HOME -batch test + - sbt/bin/sbt -java-home $JAVA_HOME -batch it:test # - sbt -Dfile.encoding=UTF8 clean coverage test coverageReport # Tricks to avoid unnecessary cache updates - find $HOME/.sbt -name "*.lock" | xargs rm - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm -#after_success: -# - bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 756d4cb8a..1be2dcdf7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,19 +1,10 @@ # Contributing to RasterFrames -Thanks for your interest in this project. +Community contributions are always welcome. To get started, please review this document, +the [code of conduct](https://github.com/locationtech/rasterframes/blob/develop/CODE_OF_CONDUCT.md), and reach out to +us on [gitter](https://gitter.im/locationtech/rasterframes) so the community can help you get started! -## Project Description - -LocationTech RasterFrames brings the power of Spark DataFrames to geospatial -raster data, empowered by the map algebra and tile layer operations of -GeoTrellis. The underlying purpose of RasterFrames is to allow data scientists -and software developers to process and analyze geospatial-temporal raster data -with the same flexibility and ease as any other Spark Catalyst data type. At its -core is a user-defined type (UDT) called TileUDT, which encodes a GeoTrellis -Tile in a form the Spark Catalyst engine can process. Furthermore, we extend the -definition of a DataFrame to encompass some additional invariants, allowing for -geospatial operations within and between RasterFrames to occur, while still -maintaining necessary geo-referencing constructs. +The official home of RasterFrames under the Eclipse Foundation may be found here: * https://projects.eclipse.org/projects/locationtech.rasterframes @@ -58,19 +49,10 @@ commands are as follows: * Build documentation: `sbt makeSite` * Spark shell with RasterFrames initialized: `sbt console` - -## Contribution Process - -RasterFrames uses GitHub pull requests (PRs) for accepting contributions. -Please fork the repository, create a branch, and submit a PR based off the `master` branch. -During the PR review process comments may be attached. Please look out for comments -and respond as necessary. - - ## Contact Help, questions and community dialog are supported via Gitter: - * https://gitter.im/s22s/raster-frames + * https://gitter.im/locationtech/rasterframes Commercial support is available by writing to info@astraea.earth diff --git a/LICENSE b/LICENSE index 152d8c948..53d10c005 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ This software is licensed under the Apache 2 license, quoted below. -Copyright 2017-2018 Astraea, Inc. +Copyright 2017-2019 Astraea, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of diff --git a/README.md b/README.md index dddeb94ae..2b3bcb43f 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,67 @@ -™ +® - [![Join the chat at https://gitter.im/s22s/raster-frames](https://badges.gitter.im/s22s/raster-frames.svg)](https://gitter.im/s22s/raster-frames?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + [![Join the chat at https://gitter.im/locationtech/rasterframes](https://badges.gitter.im/locationtech/rasterframes.svg)](https://gitter.im/locationtech/rasterframes?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -_RasterFrames™_ brings the power of Spark DataFrames to geospatial raster data, empowered by the map algebra and tile layer operations of [GeoTrellis](https://geotrellis.io/). +RasterFrames® brings together Earth-observation (EO) data access, cloud computing, and DataFrame-based data science. The recent explosion of EO data from public and private satellite operators presents both a huge opportunity as well as a challenge to the data analysis community. It is _Big Data_ in the truest sense, and its footprint is rapidly getting bigger. - +RasterFrames provides a DataFrame-centric view over arbitrary raster data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of Spark ML algorithms. By using DataFrames as the core cognitive and compute data model, it is able to deliver these features in a form that is both accessible to general analysts and scalable along with the rapidly growing data footprint. + + Please see the [Getting Started](http://rasterframes.io/getting-started.html) section of the Users' Manual to start using RasterFrames. -## Documentation +## User Resources + +* [RasterFrames Users' Manual](http://rasterframes.io/) +* [RasterFrames Jupyter Notebook Docker Image](https://hub.docker.com/r/s22s/rasterframes-notebook/) +* [Gitter Channel](https://gitter.im/locationtech/rasterframes) +* [Submit an Issue](https://github.com/locationtech/rasterframes/issues) + + +## Contributing + +Community contributions are always welcome. To get started, please review our [contribution guidelines](https://github.com/locationtech/rasterframes/blob/develop/CONTRIBUTING.md), [code of conduct](https://github.com/locationtech/rasterframes/blob/develop/CODE_OF_CONDUCT.md), and reach out to us on [gitter](https://gitter.im/locationtech/rasterframes) so the community can help you get started! + +RasterFrames is part of the LocationTech Stack. + + + +It is written in Scala, but with Python bindings. If you wish to contribute to the development of RasterFrames, or you +wish to build it from scratch, you will need [sbt](https://www.scala-sbt.org/). Then clone the repository from GitHub. + +```bash +git clone https://github.com/locationtech/rasterframes.git +cd rasterframes +``` + +To publish to your local repository: + +```bash +sbt publishLocal +``` + +You can run tests with + +```bash +sbt test +``` + +and integration tests + +```bash +sbt it:test +``` + +The documentation may be built with + +```bash +sbt makeSite +``` -* [Users' Manual](http://rasterframes.io/) -* [API Documentation](http://rasterframes.io/latest/api/index.html) -* [List of available UDFs](http://rasterframes.io/latest/api/index.html#astraea.spark.rasterframes.RasterFunctions) -* [RasterFrames Jupyter Notebook Docker Image](https://hub.docker.com/r/s22s/rasterframes-notebooks/) +Additional, Python sepcific build instruction may be found at [pyrasterframes/src/main/python/README.md](pyrasterframes/src/main/python/README.md) ## Copyright and License -RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2018. +RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2019. diff --git a/bench/archive/jmh-results-20190528095237.json b/bench/archive/jmh-results-20190528095237.json new file mode 100644 index 000000000..11d8f15f0 --- /dev/null +++ b/bench/archive/jmh-results-20190528095237.json @@ -0,0 +1,163 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEquals", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 421.72586734818816, + "scoreError" : 21.639640737136855, + "scoreConfidence" : [ + 400.0862266110513, + 443.365508085325 + ], + "scorePercentiles" : { + "0.0" : 416.0271304058273, + "50.0" : 420.27062086802925, + "90.0" : 430.09766576285773, + "95.0" : 430.09766576285773, + "99.0" : 430.09766576285773, + "99.9" : 430.09766576285773, + "99.99" : 430.09766576285773, + "99.999" : 430.09766576285773, + "99.9999" : 430.09766576285773, + "100.0" : 430.09766576285773 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 417.8839821936131, + 416.0271304058273, + 430.09766576285773, + 424.349937510613, + 420.27062086802925 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.resolveCRS", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 358.41316103726996, + "scoreError" : 21.21533668234991, + "scoreConfidence" : [ + 337.19782435492004, + 379.6284977196199 + ], + "scorePercentiles" : { + "0.0" : 351.9024799408263, + "50.0" : 356.47836323413975, + "90.0" : 364.61463628732025, + "95.0" : 364.61463628732025, + "99.0" : 364.61463628732025, + "99.9" : 364.61463628732025, + "99.99" : 364.61463628732025, + "99.999" : 364.61463628732025, + "99.9999" : 364.61463628732025, + "100.0" : 364.61463628732025 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 356.47836323413975, + 364.61463628732025, + 355.393162879192, + 363.67716284487153, + 351.9024799408263 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.selfEquals", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 431.7566794354569, + "scoreError" : 27.353383248644004, + "scoreConfidence" : [ + 404.4032961868129, + 459.11006268410085 + ], + "scorePercentiles" : { + "0.0" : 426.33479529843424, + "50.0" : 428.0558107251467, + "90.0" : 443.6223317651236, + "95.0" : 443.6223317651236, + "99.0" : 443.6223317651236, + "99.9" : 443.6223317651236, + "99.99" : 443.6223317651236, + "99.999" : 443.6223317651236, + "99.9999" : 443.6223317651236, + "100.0" : 443.6223317651236 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 443.6223317651236, + 433.04934952339687, + 426.33479529843424, + 427.72110986518294, + 428.0558107251467 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/archive/jmh-results-20190528102317.json b/bench/archive/jmh-results-20190528102317.json new file mode 100644 index 000000000..20e5712a1 --- /dev/null +++ b/bench/archive/jmh-results-20190528102317.json @@ -0,0 +1,163 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEquals", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 7.084784138200969, + "scoreError" : 0.17086123184222066, + "scoreConfidence" : [ + 6.913922906358748, + 7.255645370043189 + ], + "scorePercentiles" : { + "0.0" : 7.044718220088579, + "50.0" : 7.07291741895685, + "90.0" : 7.153508778664247, + "95.0" : 7.153508778664247, + "99.0" : 7.153508778664247, + "99.9" : 7.153508778664247, + "99.99" : 7.153508778664247, + "99.999" : 7.153508778664247, + "99.9999" : 7.153508778664247, + "100.0" : 7.153508778664247 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 7.07291741895685, + 7.044718220088579, + 7.101541478172196, + 7.153508778664247, + 7.051234795122972 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.resolveCRS", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.07204597602344914, + "scoreError" : 0.014595696186190624, + "scoreConfidence" : [ + 0.05745027983725852, + 0.08664167220963977 + ], + "scorePercentiles" : { + "0.0" : 0.067949003349235, + "50.0" : 0.07168162461398803, + "90.0" : 0.07812749538776566, + "95.0" : 0.07812749538776566, + "99.0" : 0.07812749538776566, + "99.9" : 0.07812749538776566, + "99.99" : 0.07812749538776566, + "99.999" : 0.07812749538776566, + "99.9999" : 0.07812749538776566, + "100.0" : 0.07812749538776566 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.0701740653091496, + 0.07812749538776566, + 0.07229769145710743, + 0.07168162461398803, + 0.067949003349235 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.selfEquals", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.04160068241214439, + "scoreError" : 8.000367242705733E-4, + "scoreConfidence" : [ + 0.04080064568787382, + 0.04240071913641496 + ], + "scorePercentiles" : { + "0.0" : 0.0413462799613575, + "50.0" : 0.04153631854728124, + "90.0" : 0.04183927734451199, + "95.0" : 0.04183927734451199, + "99.0" : 0.04183927734451199, + "99.9" : 0.04183927734451199, + "99.99" : 0.04183927734451199, + "99.999" : 0.04183927734451199, + "99.9999" : 0.04183927734451199, + "100.0" : 0.04183927734451199 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.0413462799613575, + 0.04183927734451199, + 0.04153631854728124, + 0.0417885245223439, + 0.0414930116852273 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/archive/jmh-results-20190606082738.json b/bench/archive/jmh-results-20190606082738.json new file mode 100644 index 000000000..b8e7467b1 --- /dev/null +++ b/bench/archive/jmh-results-20190606082738.json @@ -0,0 +1,269 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEqualsFalse", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 13.115296907959536, + "scoreError" : 2.5400077963191556, + "scoreConfidence" : [ + 10.57528911164038, + 15.655304704278691 + ], + "scorePercentiles" : { + "0.0" : 12.590614558661818, + "50.0" : 12.853830352008682, + "90.0" : 14.25982363939229, + "95.0" : 14.25982363939229, + "99.0" : 14.25982363939229, + "99.9" : 14.25982363939229, + "99.99" : 14.25982363939229, + "99.999" : 14.25982363939229, + "99.9999" : 14.25982363939229, + "100.0" : 14.25982363939229 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 12.853830352008682, + 12.590614558661818, + 12.829707376038487, + 14.25982363939229, + 13.042508613696407 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEqualsTrue", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 6.5973550106567345, + "scoreError" : 0.1946737881542353, + "scoreConfidence" : [ + 6.402681222502499, + 6.7920287988109695 + ], + "scorePercentiles" : { + "0.0" : 6.523477357639692, + "50.0" : 6.6063669572343695, + "90.0" : 6.648688182671118, + "95.0" : 6.648688182671118, + "99.0" : 6.648688182671118, + "99.9" : 6.648688182671118, + "99.99" : 6.648688182671118, + "99.999" : 6.648688182671118, + "99.9999" : 6.648688182671118, + "100.0" : 6.648688182671118 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 6.635409622463296, + 6.523477357639692, + 6.648688182671118, + 6.572832933275196, + 6.6063669572343695 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalLazyEqualsFalse", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 13.265730662256157, + "scoreError" : 1.915211216125259, + "scoreConfidence" : [ + 11.350519446130898, + 15.180941878381416 + ], + "scorePercentiles" : { + "0.0" : 12.850610015459289, + "50.0" : 13.13459015560355, + "90.0" : 14.120588306765669, + "95.0" : 14.120588306765669, + "99.0" : 14.120588306765669, + "99.9" : 14.120588306765669, + "99.99" : 14.120588306765669, + "99.999" : 14.120588306765669, + "99.9999" : 14.120588306765669, + "100.0" : 14.120588306765669 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 13.215365538647765, + 13.13459015560355, + 13.007499294804513, + 12.850610015459289, + 14.120588306765669 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalLazyEqualsTrue", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.040409137130485946, + "scoreError" : 0.007586963982726796, + "scoreConfidence" : [ + 0.03282217314775915, + 0.047996101113212744 + ], + "scorePercentiles" : { + "0.0" : 0.03867633630965359, + "50.0" : 0.04003336568127626, + "90.0" : 0.04377238392008154, + "95.0" : 0.04377238392008154, + "99.0" : 0.04377238392008154, + "99.9" : 0.04377238392008154, + "99.99" : 0.04377238392008154, + "99.999" : 0.04377238392008154, + "99.9999" : 0.04377238392008154, + "100.0" : 0.04377238392008154 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.03867633630965359, + 0.04003336568127626, + 0.04016005719940341, + 0.03940354254201491, + 0.04377238392008154 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.resolveCRS", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.06507083680791029, + "scoreError" : 0.003209489715829842, + "scoreConfidence" : [ + 0.061861347092080445, + 0.06828032652374012 + ], + "scorePercentiles" : { + "0.0" : 0.06425209212071442, + "50.0" : 0.06461825090771647, + "90.0" : 0.06612649264562556, + "95.0" : 0.06612649264562556, + "99.0" : 0.06612649264562556, + "99.9" : 0.06612649264562556, + "99.99" : 0.06612649264562556, + "99.999" : 0.06612649264562556, + "99.9999" : 0.06612649264562556, + "100.0" : 0.06612649264562556 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.06612649264562556, + 0.06579754694820603, + 0.06461825090771647, + 0.06425209212071442, + 0.06455980141728893 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/archive/jmh-results-20190606094001.json b/bench/archive/jmh-results-20190606094001.json new file mode 100644 index 000000000..02cdd3194 --- /dev/null +++ b/bench/archive/jmh-results-20190606094001.json @@ -0,0 +1,269 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEqualsFalse", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 13.076857373851485, + "scoreError" : 1.6593497203225103, + "scoreConfidence" : [ + 11.417507653528975, + 14.736207094173995 + ], + "scorePercentiles" : { + "0.0" : 12.633328785860648, + "50.0" : 13.200439575276704, + "90.0" : 13.659196200240215, + "95.0" : 13.659196200240215, + "99.0" : 13.659196200240215, + "99.9" : 13.659196200240215, + "99.99" : 13.659196200240215, + "99.999" : 13.659196200240215, + "99.9999" : 13.659196200240215, + "100.0" : 13.659196200240215 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 13.659196200240215, + 12.665249239331997, + 13.200439575276704, + 13.226073068547855, + 12.633328785860648 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalEqualsTrue", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.2775587837304895, + "scoreError" : 0.015896893581796353, + "scoreConfidence" : [ + 0.2616618901486931, + 0.29345567731228583 + ], + "scorePercentiles" : { + "0.0" : 0.2724269842972383, + "50.0" : 0.2775487008943729, + "90.0" : 0.2816631615036355, + "95.0" : 0.2816631615036355, + "99.0" : 0.2816631615036355, + "99.9" : 0.2816631615036355, + "99.99" : 0.2816631615036355, + "99.999" : 0.2816631615036355, + "99.9999" : 0.2816631615036355, + "100.0" : 0.2816631615036355 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.28157426698598376, + 0.2816631615036355, + 0.27458080497121706, + 0.2775487008943729, + 0.2724269842972383 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalLazyEqualsFalse", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 13.143184640034391, + "scoreError" : 0.6865674500293741, + "scoreConfidence" : [ + 12.456617190005018, + 13.829752090063765 + ], + "scorePercentiles" : { + "0.0" : 12.925994808467195, + "50.0" : 13.17768387931118, + "90.0" : 13.341295384511856, + "95.0" : 13.341295384511856, + "99.0" : 13.341295384511856, + "99.9" : 13.341295384511856, + "99.99" : 13.341295384511856, + "99.999" : 13.341295384511856, + "99.9999" : 13.341295384511856, + "100.0" : 13.341295384511856 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 13.341295384511856, + 13.17768387931118, + 12.925994808467195, + 12.995056782282637, + 13.27589234559909 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.logicalLazyEqualsTrue", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.03659740987415034, + "scoreError" : 0.0011385555881718446, + "scoreConfidence" : [ + 0.035458854285978496, + 0.037735965462322184 + ], + "scorePercentiles" : { + "0.0" : 0.036314017083098636, + "50.0" : 0.0364980924818408, + "90.0" : 0.03691511396572689, + "95.0" : 0.03691511396572689, + "99.0" : 0.03691511396572689, + "99.9" : 0.03691511396572689, + "99.99" : 0.03691511396572689, + "99.999" : 0.03691511396572689, + "99.9999" : 0.03691511396572689, + "100.0" : 0.03691511396572689 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.036314017083098636, + 0.03635019813669222, + 0.0364980924818408, + 0.03691511396572689, + 0.03690962770339316 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CRSBench.resolveCRS", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx2048M", + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.06517305542168148, + "scoreError" : 0.005310236878903678, + "scoreConfidence" : [ + 0.05986281854277779, + 0.07048329230058516 + ], + "scorePercentiles" : { + "0.0" : 0.06355938701044708, + "50.0" : 0.06512777854120488, + "90.0" : 0.06700762802360496, + "95.0" : 0.06700762802360496, + "99.0" : 0.06700762802360496, + "99.9" : 0.06700762802360496, + "99.99" : 0.06700762802360496, + "99.999" : 0.06700762802360496, + "99.9999" : 0.06700762802360496, + "100.0" : 0.06700762802360496 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.06700762802360496, + 0.06512777854120488, + 0.06418521981870606, + 0.06598526371444442, + 0.06355938701044708 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/src/main/resources/log4j.properties b/bench/src/main/resources/log4j.properties index f8cb2c5b9..9ed1d66ca 100644 --- a/bench/src/main/resources/log4j.properties +++ b/bench/src/main/resources/log4j.properties @@ -18,7 +18,7 @@ # # Set everything to be logged to the console -log4j.rootCategory=TRACE, FILE +log4j.rootCategory=INFO, FILE log4j.appender.FILE=org.apache.log4j.FileAppender log4j.appender.FILE.File=target/jmh-log.out @@ -36,7 +36,7 @@ log4j.logger.org.spark_project.jetty=WARN log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO -log4j.logger.astraea.spark=DEBUG +log4j.logger.org.locationtech=DEBUG # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/RasterRefBench.scala b/bench/src/main/scala/astraea/spark/rasterframes/bench/RasterRefBench.scala deleted file mode 100644 index c68c826e8..000000000 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/RasterRefBench.scala +++ /dev/null @@ -1,129 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.bench - - -import java.util.concurrent.TimeUnit - -import astraea.spark.rasterframes -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.transformers.RasterSourceToTiles -import astraea.spark.rasterframes.ref.RasterSource -import astraea.spark.rasterframes.ref.RasterSource.ReadCallback -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.sql._ -import org.openjdk.jmh.annotations._ -/** - * - * - * @since 11/1/18 - */ -@BenchmarkMode(Array(Mode.AverageTime)) -@State(Scope.Benchmark) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -class RasterRefBench extends SparkEnv with LazyLogging { - import spark.implicits._ - - var expandedDF: DataFrame = _ - var singleDF: DataFrame = _ - - @Setup(Level.Trial) - def setupData(): Unit = { - val watcher = new ReadCallback { - var count: Long = 0 - var calls: Int = 0 - override def readRange(source: RasterSource, start: Long, length: Int): Unit = { - calls += 1 - count += length - logger.debug("%4d -- %,d bytes".format(calls, count)) - } - } - - val r1 = RasterSource(remoteCOGSingleband1, Some(watcher)) - val r2 = RasterSource(remoteCOGSingleband2, Some(watcher)) - singleDF = Seq((r1, r2)).toDF("B1", "B2") - .select(RasterSourceToTiles(false, $"B1", $"B2")) - - expandedDF = Seq((r1, r2)).toDF("B1", "B2") - .select(RasterSourceToTiles(true, $"B1", $"B2")) - } - - @Benchmark - def computeDifferenceExpanded() = { - expandedDF - .select(normalized_difference($"B1", $"B2")) - .cache() - .count() - } - - @Benchmark - def computeDifferenceSingle() = { - singleDF - .select(normalized_difference($"B1", $"B2")) - .cache() - .count() - } - - @Benchmark - def computeStatsSingle() = { - singleDF.select(agg_stats($"B1")).collect() - } - - @Benchmark - def computeStatsExpanded() = { - expandedDF.select(agg_stats($"B1")).collect() - } - - @Benchmark - def computeDifferenceStats() = { - singleDF.select(agg_stats(normalized_difference($"B1", $"B2"))).collect() - } - -} - -object RasterRefBench { - -// import org.openjdk.jmh.runner.RunnerException -// import org.openjdk.jmh.runner.options.OptionsBuilder -// -// @throws[RunnerException] - def main(args: Array[String]): Unit = { - - val thing = new RasterRefBench() - thing.setupData() - rasterframes.util.time("compute stats expanded") { - thing.computeStatsSingle() - } - - rasterframes.util.time("compute stats single") { - thing.computeStatsExpanded() - } - - // val opt = new OptionsBuilder() -// .include(classOf[RasterRefBench].getSimpleName) -// .threads(4) -// .forks(5) -// .build() -// -// new Runner(opt).run() - } -} diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/BinaryTileOpBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/BinaryTileOpBench.scala similarity index 92% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/BinaryTileOpBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/BinaryTileOpBench.scala index 133d93356..dce7a7715 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/BinaryTileOpBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/BinaryTileOpBench.scala @@ -19,11 +19,12 @@ * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench + import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes.expressions.localops._ -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes._ import geotrellis.raster.Tile import geotrellis.raster.mapalgebra.{local => gt} import org.apache.spark.sql._ diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CRSBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CRSBench.scala new file mode 100644 index 000000000..424533b58 --- /dev/null +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/CRSBench.scala @@ -0,0 +1,69 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.bench + +import java.util.concurrent.TimeUnit + +import geotrellis.proj4.{CRS, LatLng, WebMercator} +import org.locationtech.proj4j.CoordinateReferenceSystem +import org.locationtech.rasterframes.model.LazyCRS +import org.openjdk.jmh.annotations._ + +@BenchmarkMode(Array(Mode.AverageTime)) +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +class CRSBench extends SparkEnv { + + var crs1: CRS = _ + var crs2: CRS = _ + + @Setup(Level.Invocation) + def setupData(): Unit = { + crs1 = LazyCRS("epsg:4326") + crs2 = LazyCRS(WebMercator.toProj4String) + } + + @Benchmark + def resolveCRS(): CoordinateReferenceSystem = { + crs1.proj4jCrs + } + + @Benchmark + def logicalEqualsTrue(): Boolean = { + crs1 == LatLng + } + + @Benchmark + def logicalEqualsFalse(): Boolean = { + crs1 == WebMercator + } + + @Benchmark + def logicalLazyEqualsTrue(): Boolean = { + crs1 == crs1 + } + + @Benchmark + def logicalLazyEqualsFalse(): Boolean = { + crs1 == crs2 + } +} diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/CatalystSerializerBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala similarity index 94% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/CatalystSerializerBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala index b4abad9bf..12a6b0486 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/CatalystSerializerBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala @@ -19,15 +19,15 @@ * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes.encoders.{CatalystSerializer, StandardEncoders} import geotrellis.proj4.{CRS, LatLng, Sinusoidal} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.locationtech.rasterframes.encoders.{CatalystSerializer, StandardEncoders} import org.openjdk.jmh.annotations._ @BenchmarkMode(Array(Mode.AverageTime)) diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/MultibandRenderBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/MultibandRenderBench.scala similarity index 87% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/MultibandRenderBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/MultibandRenderBench.scala index 383710205..8636a6a80 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/MultibandRenderBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/MultibandRenderBench.scala @@ -19,12 +19,13 @@ * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench + import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes.util.MultibandRender.Landsat8NaturalColor +import org.locationtech.rasterframes.util.MultibandRender.Landsat8NaturalColor import geotrellis.raster._ -import geotrellis.raster.io.geotiff.{GeoTiff, MultibandGeoTiff} +import geotrellis.raster.io.geotiff.MultibandGeoTiff import org.apache.commons.io.IOUtils import org.openjdk.jmh.annotations._ diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala new file mode 100644 index 000000000..448fab9c3 --- /dev/null +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala @@ -0,0 +1,87 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.bench + +import java.util.concurrent.TimeUnit + +import com.typesafe.scalalogging.LazyLogging +import org.apache.spark.sql._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs +import org.locationtech.rasterframes.expressions.transformers.RasterRefToTile +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.ref.RasterSource +import org.openjdk.jmh.annotations._ + +@BenchmarkMode(Array(Mode.AverageTime)) +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +class RasterRefBench extends SparkEnv with LazyLogging { + import spark.implicits._ + + var expandedDF: DataFrame = _ + var singleDF: DataFrame = _ + + @Setup(Level.Trial) + def setupData(): Unit = { + val r1 = RasterSource(remoteCOGSingleband1) + val r2 = RasterSource(remoteCOGSingleband2) + + singleDF = Seq((r1, r2)).toDF("B1", "B2") + .select(RasterRefToTile(RasterSourceToRasterRefs(Some(TileDimensions(r1.dimensions)), Seq(0), $"B1", $"B2"))) + + expandedDF = Seq((r1, r2)).toDF("B1", "B2") + .select(RasterRefToTile(RasterSourceToRasterRefs($"B1", $"B2"))) + } + + @Benchmark + def computeDifferenceExpanded() = { + expandedDF + .select(rf_normalized_difference($"B1", $"B2")) + .cache() + .count() + } + + @Benchmark + def computeDifferenceSingle() = { + singleDF + .select(rf_normalized_difference($"B1", $"B2")) + .cache() + .count() + } + + @Benchmark + def computeStatsSingle() = { + singleDF.select(rf_agg_stats($"B1")).collect() + } + + @Benchmark + def computeStatsExpanded() = { + expandedDF.select(rf_agg_stats($"B1")).collect() + } + + @Benchmark + def computeDifferenceStats() = { + singleDF.select(rf_agg_stats(rf_normalized_difference($"B1", $"B2"))).collect() + } + +} \ No newline at end of file diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/SparkEnv.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/SparkEnv.scala similarity index 92% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/SparkEnv.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/SparkEnv.scala index 8b718479a..d3691f800 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/SparkEnv.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/SparkEnv.scala @@ -15,11 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import org.apache.spark.sql.SparkSession import org.openjdk.jmh.annotations.{Level, TearDown} diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/StatsComputeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/StatsComputeBench.scala similarity index 75% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/StatsComputeBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/StatsComputeBench.scala index c9aa7eef4..2ebc3efc0 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/StatsComputeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/StatsComputeBench.scala @@ -15,14 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.stats.CellHistogram +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.stats.CellHistogram import org.apache.spark.sql._ import org.openjdk.jmh.annotations._ @@ -57,26 +59,26 @@ class StatsComputeBench extends SparkEnv { // @Benchmark // def computeStats(): Array[CellStatistics] = { -// tiles.select(agg_stats($"tile")).collect() +// tiles.select(rf_agg_stats($"tile")).collect() // } @Benchmark def computeHistogram(): Array[CellHistogram] = { - tiles.select(agg_approx_histogram($"tile")).collect() + tiles.select(rf_agg_approx_histogram($"tile")).collect() } // @Benchmark // def extractMean(): Array[Double] = { -// tiles.select(agg_stats($"tile").getField("mean")).map(_.getDouble(0)).collect() +// tiles.select(rf_agg_stats($"tile").getField("mean")).map(_.getDouble(0)).collect() // } // // @Benchmark // def directMean(): Array[Double] = { -// tiles.repartition(10).select(agg_mean($"tile")).collect() +// tiles.repartition(10).select(rf_agg_mean($"tile")).collect() // } // @Benchmark // def computeCounts() = { -// tiles.toDF("tile").select(data_cells($"tile") as "counts").agg(sum($"counts")).collect() +// tiles.toDF("tile").select(rf_data_cells($"tile") as "counts").agg(sum($"counts")).collect() // } } diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileAssembleBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileAssembleBench.scala similarity index 90% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/TileAssembleBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/TileAssembleBench.scala index 8fe31ef0d..8b33af369 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileAssembleBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileAssembleBench.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea, Inc. + * Copyright 2017 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -19,10 +19,11 @@ * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench + import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster.ByteConstantNoDataCellType import org.apache.spark.sql._ import org.openjdk.jmh.annotations._ @@ -44,7 +45,7 @@ class TileAssembleBench extends SparkEnv { var cells1: DataFrame = _ var cells2: DataFrame = _ - val assembler = assemble_tile( + val assembler = rf_assemble_tile( $"column_index", $"row_index", $"tile", tileSize, tileSize, cellType ) @@ -53,7 +54,7 @@ class TileAssembleBench extends SparkEnv { def setupData(): Unit = { cells1 = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellType.name)).zipWithIndex .toDF("tile", "id") - .select($"id", explode_tiles($"tile")) + .select($"id", rf_explode_tiles($"tile")) .repartition(4, $"id") .cache() diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileCellScanBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala similarity index 92% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/TileCellScanBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala index 64ee8716e..350ac811a 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileCellScanBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala @@ -15,15 +15,17 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes.tiles.InternalRowTile import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.rf.TileUDT +import org.locationtech.rasterframes.tiles.InternalRowTile import org.openjdk.jmh.annotations._ @BenchmarkMode(Array(Mode.AverageTime)) diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala similarity index 85% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/TileEncodeBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala index 7f25235ae..a4b0a2595 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileEncodeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala @@ -15,19 +15,22 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench import java.net.URI import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile -import astraea.spark.rasterframes.ref.{RasterRef, RasterSource} +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes.ref.RasterRef import geotrellis.raster.Tile import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.locationtech.rasterframes.ref.{RasterRef, RasterSource} import org.openjdk.jmh.annotations._ @BenchmarkMode(Array(Mode.AverageTime)) @@ -52,7 +55,7 @@ class TileEncodeBench extends SparkEnv { cellTypeName match { case "rasterRef" ⇒ val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF" - tile = RasterRefTile(RasterRef(RasterSource(URI.create(baseCOG)), Some(Extent(253785.0, 3235185.0, 485115.0, 3471015.0)))) + tile = RasterRefTile(RasterRef(RasterSource(URI.create(baseCOG)), 0, Some(Extent(253785.0, 3235185.0, 485115.0, 3471015.0)))) case _ ⇒ tile = randomTile(tileSize, tileSize, cellTypeName) } diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileExplodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala similarity index 86% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/TileExplodeBench.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala index ebd4f169c..7f3352f69 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/TileExplodeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala @@ -15,12 +15,14 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.bench +package org.locationtech.rasterframes.bench import java.util.concurrent.TimeUnit -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.openjdk.jmh.annotations._ @@ -56,11 +58,11 @@ class TileExplodeBench extends SparkEnv { @Benchmark def arrayExplode() = { - tiles.select(posexplode(tile_to_array_double($"tile"))).count() + tiles.select(posexplode(rf_tile_to_array_double($"tile"))).count() } @Benchmark def tileExplode() = { - tiles.select(explode_tiles($"tile")).count() + tiles.select(rf_explode_tiles($"tile")).count() } } diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/package.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala similarity index 97% rename from bench/src/main/scala/astraea/spark/rasterframes/bench/package.scala rename to bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala index 525c86734..65d8ab88f 100644 --- a/bench/src/main/scala/astraea/spark/rasterframes/bench/package.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala @@ -17,7 +17,7 @@ * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes import java.net.URI diff --git a/build.sbt b/build.sbt index 05aed4e8d..5aa2e6009 100644 --- a/build.sbt +++ b/build.sbt @@ -1,46 +1,163 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017-2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + addCommandAlias("makeSite", "docs/makeSite") +addCommandAlias("previewSite", "docs/previewSite") +addCommandAlias("ghpagesPushSite", "docs/ghpagesPushSite") addCommandAlias("console", "datasource/console") +// Prefer our own IntegrationTest config definition, which inherits from Test. +lazy val IntegrationTest = config("it") extend Test + lazy val root = project .in(file(".")) .withId("RasterFrames") .aggregate(core, datasource, pyrasterframes, experimental) + .enablePlugins(RFReleasePlugin) .settings(publish / skip := true) - .settings(releaseSettings) - -lazy val deployment = project - .dependsOn(root) - .disablePlugins(SparkPackagePlugin) -lazy val IntegrationTest = config("it") extend Test +lazy val `rf-notebook` = project + .dependsOn(pyrasterframes) + .enablePlugins(RFAssemblyPlugin, DockerPlugin) + .settings(publish / skip := true) lazy val core = project + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings(inConfig(IntegrationTest)(Defaults.testSettings)) .settings(Defaults.itSettings) - .disablePlugins(SparkPackagePlugin) + .settings( + moduleName := "rasterframes", + libraryDependencies ++= Seq( + shapeless, + `jts-core`, + geomesa("z3").value, + geomesa("spark-jts").value, + `geotrellis-contrib-vlm`, + `geotrellis-contrib-gdal`, + spark("core").value % Provided, + spark("mllib").value % Provided, + spark("sql").value % Provided, + geotrellis("spark").value, + geotrellis("raster").value, + geotrellis("s3").value, + geotrellis("spark-testkit").value % Test excludeAll ( + ExclusionRule(organization = "org.scalastic"), + ExclusionRule(organization = "org.scalatest") + ), + scaffeine, + scalatest + ), + buildInfoKeys ++= Seq[BuildInfoKey]( + moduleName, version, scalaVersion, sbtVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion + ), + buildInfoPackage := "org.locationtech.rasterframes", + buildInfoObject := "RFBuildInfo", + buildInfoOptions := Seq( + BuildInfoOption.ToMap, + BuildInfoOption.BuildTime, + BuildInfoOption.ToJson + ) + ) lazy val pyrasterframes = project .dependsOn(core, datasource, experimental) - .settings(assemblySettings) + .enablePlugins(RFAssemblyPlugin, PythonBuildPlugin) + .settings( + libraryDependencies ++= Seq( + geotrellis("s3").value, + spark("core").value % Provided, + spark("mllib").value % Provided, + spark("sql").value % Provided + ) + ) lazy val datasource = project + .configs(IntegrationTest) + .settings(Defaults.itSettings) .dependsOn(core % "test->test;compile->compile") - .disablePlugins(SparkPackagePlugin) + .settings( + moduleName := "rasterframes-datasource", + libraryDependencies ++= Seq( + geotrellis("s3").value, + spark("core").value % Provided, + spark("mllib").value % Provided, + spark("sql").value % Provided + ), + initialCommands in console := (initialCommands in console).value + + """ + |import org.locationtech.rasterframes.datasource.geotrellis._ + |import org.locationtech.rasterframes.datasource.geotiff._ + |""".stripMargin + ) lazy val experimental = project .configs(IntegrationTest) .settings(Defaults.itSettings) .dependsOn(core % "test->test;it->test;compile->compile") .dependsOn(datasource % "test->test;it->test;compile->compile") - .disablePlugins(SparkPackagePlugin) + .settings( + moduleName := "rasterframes-experimental", + libraryDependencies ++= Seq( + geotrellis("s3").value, + spark("core").value % Provided, + spark("mllib").value % Provided, + spark("sql").value % Provided + ), + fork in IntegrationTest := true, + javaOptions in IntegrationTest := Seq("-Xmx2G"), + parallelExecution in IntegrationTest := false + ) lazy val docs = project - .dependsOn(core, datasource) - .disablePlugins(SparkPackagePlugin) + .dependsOn(core, datasource, pyrasterframes) + .enablePlugins(SiteScaladocPlugin, ParadoxPlugin, GhpagesPlugin, ScalaUnidocPlugin) + .settings( + apiURL := Some(url("http://rasterframes.io/latest/api")), + autoAPIMappings := true, + ghpagesNoJekyll := true, + ScalaUnidoc / siteSubdirName := "latest/api", + paradox / siteSubdirName := ".", + paradoxProperties ++= Map( + "github.base_url" -> "https://github.com/locationtech/rasterframes", + "version" -> version.value, + "scaladoc.org.apache.spark.sql.rf" -> "http://rasterframes.io/latest" + ), + paradoxNavigationExpandDepth := Some(3), + paradoxTheme := Some(builtinParadoxTheme("generic")), + makeSite := makeSite + .dependsOn(Compile / unidoc) + .dependsOn((Compile / paradox) + .dependsOn(pyrasterframes / doc) + ).value, + Compile / paradox / sourceDirectories += (pyrasterframes / Python / doc / target).value, + ) + .settings( + addMappingsToSiteDir(ScalaUnidoc / packageDoc / mappings, ScalaUnidoc / siteSubdirName) + ) + .settings( + addMappingsToSiteDir(Compile / paradox / mappings, paradox / siteSubdirName) + ) lazy val bench = project .dependsOn(core % "compile->test") - .disablePlugins(SparkPackagePlugin) .settings(publish / skip := true) diff --git a/build/circleci/Dockerfile b/build/circleci/Dockerfile new file mode 100644 index 000000000..334c1b15f --- /dev/null +++ b/build/circleci/Dockerfile @@ -0,0 +1,74 @@ +FROM circleci/openjdk:8-jdk + +ENV OPENJPEG_VERSION 2.3.0 +ENV GDAL_VERSION 2.4.1 +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +# most of these libraries required for +# python-pip pandoc && pip install setuptools => required for pyrasterframes testing +RUN sudo apt-get update && \ + sudo apt remove \ + python python-minimal python2.7 python2.7-minimal \ + libpython-stdlib libpython2.7 libpython2.7-minimal libpython2.7-stdlib \ + && sudo apt-get install -y \ + python3 \ + python3-pip \ + pandoc \ + wget \ + gcc g++ build-essential \ + libcurl4-gnutls-dev \ + libproj-dev \ + libgeos-dev \ + libhdf4-alt-dev \ + libhdf5-serial-dev \ + bash-completion \ + cmake \ + imagemagick \ + libpng-dev \ + swig \ + ant \ + && sudo apt autoremove \ + && sudo apt-get clean all \ + && pip3 install setuptools ipython==6.2.1 \ + && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +# install OpenJPEG +RUN cd /tmp && \ + wget https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz && \ + tar -xf v${OPENJPEG_VERSION}.tar.gz && \ + cd openjpeg-${OPENJPEG_VERSION}/ && \ + mkdir build && \ + cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ && \ + make -j && \ + sudo make install && \ + cd /tmp && rm -Rf v${OPENJPEG_VERSION}.tar.gz openjpeg* + +# Compile and install GDAL with Java bindings +RUN cd /tmp && \ + wget http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz && \ + tar -xf gdal-${GDAL_VERSION}.tar.gz && \ + cd gdal-${GDAL_VERSION} && \ + ./configure \ + --with-curl \ + --with-hdf4 \ + --with-hdf5 \ + --with-geos \ + --with-geotiff=internal \ + --with-hide-internal-symbols \ + --with-java=$JAVA_HOME \ + --with-libtiff=internal \ + --with-libz=internal \ + --with-mrf \ + --with-openjpeg \ + --with-threads \ + --without-jp2mrsid \ + --without-netcdf \ + --without-ecw \ + && \ + make -j 8 && \ + sudo make install && \ + cd swig/java && \ + sudo make install && \ + sudo ldconfig && \ + cd /tmp && sudo rm -Rf gdal* diff --git a/build/circleci/README.md b/build/circleci/README.md new file mode 100644 index 000000000..69b9cdff3 --- /dev/null +++ b/build/circleci/README.md @@ -0,0 +1,6 @@ +# CircleCI Dockerfile Build file + +```bash +docker build -t s22s/rasterframes-circleci:latest . +docker push s22s/rasterframes-circleci:latest +``` diff --git a/core/build.sbt b/core/build.sbt deleted file mode 100644 index eb5164045..000000000 --- a/core/build.sbt +++ /dev/null @@ -1,35 +0,0 @@ -enablePlugins(BuildInfoPlugin) - -moduleName := "rasterframes" - -libraryDependencies ++= Seq( - "com.chuusai" %% "shapeless" % "2.3.2", - "org.locationtech.geomesa" %% "geomesa-z3" % rfGeoMesaVersion.value, - "org.locationtech.geomesa" %% "geomesa-spark-jts" % rfGeoMesaVersion.value exclude("jgridshift", "jgridshift"), - - spark("core").value % Provided, - spark("mllib").value % Provided, - spark("sql").value % Provided, - geotrellis("spark").value, - geotrellis("raster").value, - geotrellis("s3").value, - geotrellis("spark-testkit").value % Test excludeAll ( - ExclusionRule(organization = "org.scalastic"), - ExclusionRule(organization = "org.scalatest") - ), - scalaTest -) - -buildInfoKeys ++= Seq[BuildInfoKey]( - name, version, scalaVersion, sbtVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion -) - -buildInfoPackage := "astraea.spark.rasterframes" - -buildInfoObject := "RFBuildInfo" - -buildInfoOptions := Seq( - BuildInfoOption.ToMap, - BuildInfoOption.BuildTime -) - diff --git a/core/src/it/resources/log4j.properties b/core/src/it/resources/log4j.properties index 378ae8e61..1135e4b34 100644 --- a/core/src/it/resources/log4j.properties +++ b/core/src/it/resources/log4j.properties @@ -37,8 +37,8 @@ log4j.logger.org.spark_project.jetty=WARN log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO -log4j.logger.astraea.spark.rasterframes=DEBUG -log4j.logger.astraea.spark.rasterframes.ref=TRACE +log4j.logger.org.locationtech.rasterframes=WARN +log4j.logger.org.locationtech.rasterframes.ref=WARN log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support diff --git a/core/src/it/scala/astraea/spark/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/astraea/spark/rasterframes/ref/RasterSourceIT.scala deleted file mode 100644 index 6f9069183..000000000 --- a/core/src/it/scala/astraea/spark/rasterframes/ref/RasterSourceIT.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2019 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.ref - -import java.net.URI - -import astraea.spark.rasterframes.TestEnvironment.ReadMonitor -import astraea.spark.rasterframes.ref.RasterSource.FileGeoTiffRasterSource -import astraea.spark.rasterframes.{TestData, TestEnvironment} -import geotrellis.raster.io.geotiff.GeoTiff -import geotrellis.vector.Extent -import org.apache.spark.sql.rf.RasterSourceUDT - -/** - * - * - * @since 8/22/18 - */ -class RasterSourceIT extends TestEnvironment with TestData { - def sub(e: Extent) = { - val c = e.center - val w = e.width - val h = e.height - Extent(c.x, c.y, c.x + w * 0.1, c.y + h * 0.1) - } - - describe("RasterSource.readAll") { - it("should return consistently ordered tiles across bands for a given scene") { - // These specific scenes exhibit the problem where we see different subtile segment ordering across the bands of a given scene. - val rURI = new URI("https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B4.TIF") - val bURI = new URI("https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B2.TIF") - - val red = RasterSource(rURI).readAll().left.get - val blue = RasterSource(bURI).readAll().left.get - - red should not be empty - red.size should equal(blue.size) - - red.map(_.dimensions) should contain theSameElementsAs blue.map(_.dimensions) - } - } -} diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala new file mode 100644 index 000000000..ae8b0b1d4 --- /dev/null +++ b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala @@ -0,0 +1,126 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.lang.Math.ceil +import java.net.URI + +import org.locationtech.rasterframes +import org.locationtech.rasterframes.util.time +import org.locationtech.rasterframes.{NOMINAL_TILE_SIZE, TestData, TestEnvironment} + +/** + * + * + * @since 8/22/18 + */ +class RasterSourceIT extends TestEnvironment with TestData { + + describe("RasterSource.readAll") { + it("should return consistently ordered tiles across bands for a given scene") { + time(s"two band comparison prefer-gdal=${ rasterframes.rfConfig.getBoolean("prefer-gdal")}") { + // These specific scenes exhibit the problem where we see different subtile segment ordering across the bands of a given scene. + val rURI = new URI( + "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B4.TIF") + val bURI = new URI( + "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B2.TIF") + val red = time("read B4") { + RasterSource(rURI).readAll() + } + val blue = time("read B2") { + RasterSource(bURI).readAll() + } + time("test empty") { + red should not be empty + } + time("compare sizes") { + red.size should equal(blue.size) + } + time("compare dimensions") { + red.map(_.dimensions) should contain theSameElementsAs blue.map(_.dimensions) + } + } + } + } + + if (GDALRasterSource.hasGDAL) { + println("GDAL version: " + GDALRasterSource.gdalVersion()) + + describe("GDAL support") { + + + it("should read JPEG2000 scene") { + RasterSource(localSentinel).readAll().flatMap(_.tile.statisticsDouble).size should be(64) + } + + it("should read small MRF scene with one band converted from MODIS HDF") { + val (expectedTileCount, _) = expectedTileCountAndBands(2400, 2400) + RasterSource(modisConvertedMrfPath).readAll().flatMap(_.tile.statisticsDouble).size should be (expectedTileCount) + } + + it("should read remote HTTP MRF scene") { + val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4) + RasterSource(remoteHttpMrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount) + } + + it("should read remote S3 MRF scene") { + val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4) + RasterSource(remoteS3MrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount) + } + } + } else { + describe("GDAL missing error support") { + it("should throw exception reading JPEG2000 scene") { + intercept[IllegalArgumentException] { + RasterSource(localSentinel) + } + } + + it("should throw exception reading MRF scene with one band converted from MODIS HDF") { + intercept[IllegalArgumentException] { + RasterSource(modisConvertedMrfPath) + } + } + + it("should throw exception reading remote HTTP MRF scene") { + intercept[IllegalArgumentException] { + RasterSource(remoteHttpMrfPath) + } + } + + it("should throw exception reading remote S3 MRF scene") { + intercept[IllegalArgumentException] { + RasterSource(remoteS3MrfPath) + } + } + } + } + + private def expectedTileCountAndBands(x:Int, y:Int, bandCount:Int = 1) = { + val imageDimensions = Seq(x.toDouble, y.toDouble) + val tilesPerBand = imageDimensions.map(x ⇒ ceil(x / NOMINAL_TILE_SIZE)).product + val bands = Range(0, bandCount) + val expectedTileCount = tilesPerBand * bands.length + (expectedTileCount, bands) + } + +} diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 980088e28..e7d3e57f5 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1,3 +1,19 @@ rasterframes { - nominal-tile-size: 256 + nominal-tile-size = 256 + prefer-gdal = true + showable-tiles = true + showable-max-cells = 20 + raster-source-cache-timeout = 120 seconds +} + +vlm.gdal { + options { + // See https://trac.osgeo.org/gdal/wiki/ConfigOptions for options + //CPL_DEBUG = "OFF" + AWS_REQUEST_PAYER = "requester" + GDAL_DISABLE_READDIR_ON_OPEN = "YES" + CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif,.tiff,.jp2,.mrf,.idx,.lrc,.mrf.aux.xml,.vrt" + } + // set this to `false` if CPL_DEBUG is `ON` + useExceptions = true } \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/MetadataKeys.scala b/core/src/main/scala/astraea/spark/rasterframes/MetadataKeys.scala deleted file mode 100644 index 2b4948798..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/MetadataKeys.scala +++ /dev/null @@ -1,13 +0,0 @@ -package astraea.spark.rasterframes - -/** - * - * @since 2/19/18 - */ -trait MetadataKeys { - /** Key under which ContextRDD metadata is stored. */ - private[rasterframes] val CONTEXT_METADATA_KEY = "_context" - - /** Key under which RasterFrame role a column plays. */ - private[rasterframes] val SPATIAL_ROLE_KEY = "_stRole" -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala b/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala deleted file mode 100644 index ff08dd44c..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala +++ /dev/null @@ -1,353 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package astraea.spark.rasterframes - -import astraea.spark.rasterframes.expressions.TileAssembler -import astraea.spark.rasterframes.expressions.accessors._ -import astraea.spark.rasterframes.expressions.aggstats._ -import astraea.spark.rasterframes.expressions.generators._ -import astraea.spark.rasterframes.expressions.localops._ -import astraea.spark.rasterframes.expressions.tilestats._ -import astraea.spark.rasterframes.expressions.transformers._ -import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics} -import astraea.spark.rasterframes.{functions => F} -import com.vividsolutions.jts.geom.{Envelope, Geometry} -import geotrellis.proj4.CRS -import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp -import geotrellis.raster.{CellType, Tile} -import org.apache.spark.annotation.Experimental -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ - -/** - * UDFs for working with Tiles in Spark DataFrames. - * - * @since 4/3/17 - */ -trait RasterFunctions { - import util._ - import PrimitiveEncoders._ - - // format: off - /** Create a row for each cell in Tile. */ - def explode_tiles(cols: Column*): Column = explode_tiles_sample(1.0, None, cols: _*) - - /** Create a row for each cell in Tile with random sampling and optional seed. */ - def explode_tiles_sample(sampleFraction: Double, seed: Option[Long], cols: Column*): Column = - ExplodeTiles(sampleFraction, seed, cols) - - /** Create a row for each cell in Tile with random sampling (no seed). */ - def explode_tiles_sample(sampleFraction: Double, cols: Column*): Column = - ExplodeTiles(sampleFraction, None, cols) - - /** Query the number of (cols, rows) in a Tile. */ - def tile_dimensions(col: Column): Column = GetDimensions(col) - - /** Extracts the bounding box of a geometry as a JTS envelope. */ - def envelope(col: Column): TypedColumn[Any, Envelope] = GetEnvelope(col) - - /** Flattens Tile into a double array. */ - def tile_to_array_double(col: Column): TypedColumn[Any, Array[Double]] = - TileToArrayDouble(col) - - /** Flattens Tile into an integer array. */ - def tile_to_array_int(col: Column): TypedColumn[Any, Array[Double]] = - TileToArrayDouble(col) - - @Experimental - /** Convert array in `arrayCol` into a Tile of dimensions `cols` and `rows`*/ - def array_to_tile(arrayCol: Column, cols: Int, rows: Int) = withAlias("array_to_tile", arrayCol)( - udf[Tile, AnyRef](F.arrayToTile(cols, rows)).apply(arrayCol) - ) - - /** Create a Tile from a column of cell data with location indexes and preform cell conversion. */ - def assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int, ct: CellType): TypedColumn[Any, Tile] = - convert_cell_type(TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows)), ct).as(cellData.columnName).as[Tile](singlebandTileEncoder) - - /** Create a Tile from a column of cell data with location indexes. */ - def assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Column, tileRows: Column): TypedColumn[Any, Tile] = - TileAssembler(columnIndex, rowIndex, cellData, tileCols, tileRows) - - /** Extract the Tile's cell type */ - def cell_type(col: Column): TypedColumn[Any, CellType] = GetCellType(col) - - /** Change the Tile's cell type */ - def convert_cell_type(col: Column, cellType: CellType): TypedColumn[Any, Tile] = - SetCellType(col, cellType) - - /** Change the Tile's cell type */ - def convert_cell_type(col: Column, cellTypeName: String): TypedColumn[Any, Tile] = - SetCellType(col, cellTypeName) - - /** Convert a bounding box structure to a Geometry type. Intented to support multiple schemas. */ - def bounds_geometry(bounds: Column): TypedColumn[Any, Geometry] = BoundsToGeometry(bounds) - - /** Assign a `NoData` value to the Tiles. */ - def with_no_data(col: Column, nodata: Double): TypedColumn[Any, Tile] = withAlias("with_no_data", col)( - udf[Tile, Tile](F.withNoData(nodata)).apply(col) - ).as[Tile] - - /** Compute the full column aggregate floating point histogram. */ - def agg_approx_histogram(col: Column): TypedColumn[Any, CellHistogram] = - HistogramAggregate(col) - - /** Compute the full column aggregate floating point statistics. */ - def agg_stats(col: Column): TypedColumn[Any, CellStatistics] = - CellStatsAggregate(col) - - /** Computes the column aggregate mean. */ - def agg_mean(col: Column) = CellMeanAggregate(col) - - /** Computes the number of non-NoData cells in a column. */ - def agg_data_cells(col: Column): TypedColumn[Any, Long] = CellCountAggregate.DataCells(col) - - /** Computes the number of NoData cells in a column. */ - def agg_no_data_cells(col: Column): TypedColumn[Any, Long] = CellCountAggregate.NoDataCells(col) - - /** Compute the Tile-wise mean */ - def tile_mean(col: Column): TypedColumn[Any, Double] = - TileMean(col) - - /** Compute the Tile-wise sum */ - def tile_sum(col: Column): TypedColumn[Any, Double] = - Sum(col) - - /** Compute the minimum cell value in tile. */ - def tile_min(col: Column): TypedColumn[Any, Double] = - TileMin(col) - - /** Compute the maximum cell value in tile. */ - def tile_max(col: Column): TypedColumn[Any, Double] = - TileMax(col) - - /** Compute TileHistogram of Tile values. */ - def tile_histogram(col: Column): TypedColumn[Any, CellHistogram] = - TileHistogram(col) - - /** Compute statistics of Tile values. */ - def tile_stats(col: Column): TypedColumn[Any, CellStatistics] = - TileStats(col) - - /** Counts the number of non-NoData cells per Tile. */ - def data_cells(tile: Column): TypedColumn[Any, Long] = - DataCells(tile) - - /** Counts the number of NoData cells per Tile. */ - def no_data_cells(tile: Column): TypedColumn[Any, Long] = - NoDataCells(tile) - - def is_no_data_tile(tile: Column): TypedColumn[Any, Boolean] = - IsNoDataTile(tile) - - /** Compute cell-local aggregate descriptive statistics for a column of Tiles. */ - def agg_local_stats(col: Column) = - LocalStatsAggregate(col) - - /** Compute the cell-wise/local max operation between Tiles in a column. */ - def agg_local_max(col: Column): TypedColumn[Any, Tile] = LocalTileOpAggregate.LocalMaxUDAF(col) - - /** Compute the cellwise/local min operation between Tiles in a column. */ - def agg_local_min(col: Column): TypedColumn[Any, Tile] = LocalTileOpAggregate.LocalMinUDAF(col) - - /** Compute the cellwise/local mean operation between Tiles in a column. */ - def agg_local_mean(col: Column): TypedColumn[Any, Tile] = LocalMeanAggregate(col) - - /** Compute the cellwise/local count of non-NoData cells for all Tiles in a column. */ - def agg_local_data_cells(col: Column): TypedColumn[Any, Tile] = LocalCountAggregate.LocalDataCellsUDAF(col) - - /** Compute the cellwise/local count of NoData cells for all Tiles in a column. */ - def agg_local_no_data_cells(col: Column): TypedColumn[Any, Tile] = LocalCountAggregate.LocalNoDataCellsUDAF(col) - - /** Cellwise addition between two Tiles or Tile and scalar column. */ - def local_add(left: Column, right: Column): TypedColumn[Any, Tile] = Add(left, right) - - /** Cellwise addition of a scalar value to a tile. */ - def local_add[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Add(tileCol, value) - - /** Cellwise subtraction between two Tiles. */ - def local_subtract(left: Column, right: Column): TypedColumn[Any, Tile] = Subtract(left, right) - - /** Cellwise subtraction of a scalar value from a tile. */ - def local_subtract[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Subtract(tileCol, value) - - /** Cellwise multiplication between two Tiles. */ - def local_multiply(left: Column, right: Column): TypedColumn[Any, Tile] = Multiply(left, right) - - /** Cellwise multiplication of a tile by a scalar value. */ - def local_multiply[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Multiply(tileCol, value) - - /** Cellwise division between two Tiles. */ - def local_divide(left: Column, right: Column): TypedColumn[Any, Tile] = Divide(left, right) - - /** Cellwise division of a tile by a scalar value. */ - def local_divide[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Divide(tileCol, value) - - /** Perform an arbitrary GeoTrellis `LocalTileBinaryOp` between two Tile columns. */ - def local_algebra(op: LocalTileBinaryOp, left: Column, right: Column): - TypedColumn[Any, Tile] = - withAlias(opName(op), left, right)( - udf[Tile, Tile, Tile](op.apply).apply(left, right) - ).as[Tile] - - /** Compute the normalized difference of two tile columns */ - def normalized_difference(left: Column, right: Column) = - NormalizedDifference(left, right) - - /** Constructor for constant tile column */ - def make_constant_tile(value: Number, cols: Int, rows: Int, cellType: String): TypedColumn[Any, Tile] = - udf(() => F.makeConstantTile(value, cols, rows, cellType)).apply().as(s"constant_$cellType").as[Tile] - - /** Alias for column of constant tiles of zero */ - def tile_zeros(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] = - udf(() => F.tileZeros(cols, rows, cellType)).apply().as(s"zeros_$cellType").as[Tile] - - /** Alias for column of constant tiles of one */ - def tile_ones(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] = - udf(() => F.tileOnes(cols, rows, cellType)).apply().as(s"ones_$cellType").as[Tile] - - /** Where the mask tile contains NODATA, replace values in the source tile with NODATA */ - def mask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] = - Mask.MaskByDefined(sourceTile, maskTile) - - /** Where the mask tile equals the mask value, replace values in the source tile with NODATA */ - def mask_by_value(sourceTile: Column, maskTile: Column, maskValue: Column): TypedColumn[Any, Tile] = - Mask.MaskByValue(sourceTile, maskTile, maskValue) - - /** Where the mask tile DOES NOT contain NODATA, replace values in the source tile with NODATA */ - def inverse_mask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] = - Mask.InverseMaskByDefined(sourceTile, maskTile) - - /** Create a tile where cells in the grid defined by cols, rows, and bounds are filled with the given value. */ - def rasterize(geometry: Column, bounds: Column, value: Column, cols: Int, rows: Int): TypedColumn[Any, Tile] = - withAlias("rasterize", geometry)( - udf(F.rasterize(_: Geometry, _: Geometry, _: Int, cols, rows)).apply(geometry, bounds, value) - ).as[Tile] - - /** Reproject a column of geometry from one CRS to another. */ - def reproject_geometry(sourceGeom: Column, srcCRS: CRS, dstCRSCol: Column): TypedColumn[Any, Geometry] = - ReprojectGeometry(sourceGeom, srcCRS, dstCRSCol) - - /** Reproject a column of geometry from one CRS to another. */ - def reproject_geometry(sourceGeom: Column, srcCRSCol: Column, dstCRS: CRS): TypedColumn[Any, Geometry] = - ReprojectGeometry(sourceGeom, srcCRSCol, dstCRS) - - /** Reproject a column of geometry from one CRS to another. */ - def reproject_geometry(sourceGeom: Column, srcCRS: CRS, dstCRS: CRS): TypedColumn[Any, Geometry] = - ReprojectGeometry(sourceGeom, srcCRS, dstCRS) - - /** Render Tile as ASCII string, for debugging purposes. */ - def render_ascii(col: Column): TypedColumn[Any, String] = - DebugRender.RenderAscii(col) - - /** Render Tile cell values as numeric values, for debugging purposes. */ - def render_matrix(col: Column): TypedColumn[Any, String] = - DebugRender.RenderMatrix(col) - - /** Cellwise less than value comparison between two tiles. */ - def local_less(left: Column, right: Column): TypedColumn[Any, Tile] = - Less(left, right) - - /** Cellwise less than value comparison between a tile and a scalar. */ - def local_less[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - Less(tileCol, value) - - /** Cellwise less than or equal to value comparison between a tile and a scalar. */ - def local_less_equal(left: Column, right: Column): TypedColumn[Any, Tile] = - LessEqual(left, right) - - /** Cellwise less than or equal to value comparison between a tile and a scalar. */ - def local_less_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - LessEqual(tileCol, value) - - /** Cellwise greater than value comparison between two tiles. */ - def local_greater(left: Column, right: Column): TypedColumn[Any, Tile] = - Greater(left, right) - - /** Cellwise greater than value comparison between a tile and a scalar. */ - def local_greater[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - Greater(tileCol, value) - - /** Cellwise greater than or equal to value comparison between two tiles. */ - def local_greater_equal(left: Column, right: Column): TypedColumn[Any, Tile] = - GreaterEqual(left, right) - - /** Cellwise greater than or equal to value comparison between a tile and a scalar. */ - def local_greater_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - GreaterEqual(tileCol, value) - - /** Cellwise equal to value comparison between two tiles. */ - def local_equal(left: Column, right: Column): TypedColumn[Any, Tile] = - Equal(left, right) - - /** Cellwise equal to value comparison between a tile and a scalar. */ - def local_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - Equal(tileCol, value) - - /** Cellwise inequality comparison between two tiles. */ - def local_unequal(left: Column, right: Column): TypedColumn[Any, Tile] = - Unequal(left, right) - - /** Cellwise inequality comparison between a tile and a scalar. */ - def local_unequal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = - Unequal(tileCol, value) - - /** Round cell values to nearest integer without chaning cell type. */ - def round(tileCol: Column): TypedColumn[Any, Tile] = - Round(tileCol) - - /** Take natural logarithm of cell values. */ - def log(tileCol: Column): TypedColumn[Any, Tile] = - Log(tileCol) - - /** Take base 10 logarithm of cell values. */ - def log10(tileCol: Column): TypedColumn[Any, Tile] = - Log10(tileCol) - - /** Take base 2 logarithm of cell values. */ - def log2(tileCol: Column): TypedColumn[Any, Tile] = - Log2(tileCol) - - /** Natural logarithm of one plus cell values. */ - def log1p(tileCol: Column): TypedColumn[Any, Tile] = - Log1p(tileCol) - - /** Exponential of cell values */ - def exp(tileCol: Column): TypedColumn[Any, Tile] = - Exp(tileCol) - - /** Ten to the power of cell values */ - def exp10(tileCol: Column): TypedColumn[Any, Tile] = - Exp10(tileCol) - - /** Two to the power of cell values */ - def exp2(tileCol: Column): TypedColumn[Any, Tile] = - Exp2(tileCol) - - /** Exponential of cell values, less one*/ - def expm1(tileCol: Column): TypedColumn[Any, Tile] = - ExpM1(tileCol) - - /** Resample tile using nearest-neighbor */ - def resample[T: Numeric](tileCol: Column, value: T) = Resample(tileCol, value) - - /** Resample tile using nearest-neighbor */ - def resample(tileCol: Column, column2: Column) = Resample(tileCol, column2) - -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/StandardColumns.scala b/core/src/main/scala/astraea/spark/rasterframes/StandardColumns.scala deleted file mode 100644 index 340b17198..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/StandardColumns.scala +++ /dev/null @@ -1,63 +0,0 @@ -package astraea.spark.rasterframes - -import java.sql.Timestamp - -import geotrellis.raster.{Tile, TileFeature} -import geotrellis.spark.{SpatialKey, TemporalKey} -import org.apache.spark.sql.functions.col -import com.vividsolutions.jts.geom.{Point => jtsPoint, Polygon => jtsPolygon} -import geotrellis.proj4.CRS -import geotrellis.vector.Extent -import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ - -/** - * Constants identifying column in most RasterFrames. - * - * @since 2/19/18 - */ -trait StandardColumns { - /** Default RasterFrame spatial column name. */ - val SPATIAL_KEY_COLUMN = col("spatial_key").as[SpatialKey] - - /** Default RasterFrame temporal column name. */ - val TEMPORAL_KEY_COLUMN = col("temporal_key").as[TemporalKey] - - /** Default RasterFrame timestamp column name */ - val TIMESTAMP_COLUMN = col("timestamp").as[Timestamp] - - - /** Default RasterFrame column name for an tile bounds value. */ - // This is a `def` because `PolygonUDT` needs to be initialized first. - def BOUNDS_COLUMN = col("bounds").as[jtsPolygon] - - /** Default RasterFrame column name for the center coordinates of the tile's bounds. */ - // This is a `def` because `PointUDT` needs to be initialized first. - def CENTER_COLUMN = col("center").as[jtsPoint] - - /** Default Extent column name. */ - def EXTENT_COLUMN = col("extent").as[Extent] - - /** Default CRS column name. */ - def CRS_COLUMN = col("crs").as[CRS] - - /** Default RasterFrame column name for an added spatial index. */ - val SPATIAL_INDEX_COLUMN = col("spatial_index").as[Long] - - /** Default RasterFrame tile column name. */ - // This is a `def` because `TileUDT` needs to be initialized first. - def TILE_COLUMN = col("tile").as[Tile] - - /** Default RasterFrame [[TileFeature.data]] column name. */ - val TILE_FEATURE_DATA_COLUMN = col("tile_data") - - /** Default GeoTiff tags column. */ - val METADATA_COLUMN = col("metadata").as[Map[String, String]] - - /** Default column index column for the cells of exploded tiles. */ - val COLUMN_INDEX_COLUMN = col("column_index").as[Int] - - /** Default teil column index column for the cells of exploded tiles. */ - val ROW_INDEX_COLUMN = col("row_index").as[Int] -} - -object StandardColumns extends StandardColumns diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToRasterRefs.scala b/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToRasterRefs.scala deleted file mode 100644 index 2581f8be5..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToRasterRefs.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2019 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.expressions.transformers - -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.ref.RasterRef -import astraea.spark.rasterframes.util._ -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.rf._ -import org.apache.spark.sql.types.{DataType, StructField, StructType} -import org.apache.spark.sql.{Column, TypedColumn} - -import scala.util.control.NonFatal - -/** - * Accepts RasterRef and generates one or more RasterRef instances representing the - * native internal sub-tiling, if any (and requested). - * - * @since 9/6/18 - */ -case class RasterSourceToRasterRefs(children: Seq[Expression], applyTiling: Boolean) extends Expression - with Generator with CodegenFallback with ExpectsInputTypes with LazyLogging { - - private val RasterSourceType = new RasterSourceUDT() - private val rasterRefSchema = CatalystSerializer[RasterRef].schema - - override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) - override def nodeName: String = "raster_source_to_raster_ref" - - override def elementSchema: StructType = StructType( - children.map(e ⇒ StructField(e.name, rasterRefSchema, false)) - ) - - override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - try { - val refs = children.map { child ⇒ - val src = RasterSourceType.deserialize(child.eval(input)) - if (applyTiling) src.nativeTiling.map(e ⇒ RasterRef(src, Some(e))) else Seq(RasterRef(src)) - } - refs.transpose.map(ts ⇒ InternalRow(ts.map(_.toInternalRow): _*)) - } - catch { - case NonFatal(ex) ⇒ - logger.error("Error fetching data for " + input, ex) - Traversable.empty - } - } -} - -object RasterSourceToRasterRefs { - def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(true, rrs: _*) - def apply(applyTiling: Boolean, rrs: Column*): TypedColumn[Any, RasterRef] = - new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), applyTiling)).as[RasterRef] -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToTiles.scala b/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToTiles.scala deleted file mode 100644 index 2b1caa3ba..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterSourceToTiles.scala +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2019 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.expressions.transformers - -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.util._ -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.rf._ -import org.apache.spark.sql.types.{DataType, StructField, StructType} - -import scala.util.control.NonFatal - -/** - * Accepts RasterRef and generates one or more RasterRef instances representing the - * native internal sub-tiling, if any (and requested). - * - * @since 9/6/18 - */ -case class RasterSourceToTiles(children: Seq[Expression], applyTiling: Boolean) extends Expression - with Generator with CodegenFallback with ExpectsInputTypes with LazyLogging { - - private val RasterSourceType = new RasterSourceUDT() - private val TileType = new TileUDT() - - override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) - override def nodeName: String = "raster_source_to_tile" - - override def elementSchema: StructType = StructType( - children.map(e ⇒ StructField(e.name, TileType, true)) - ) - - override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - implicit val ser = TileUDT.tileSerializer - - try { - val refs = children.map { child ⇒ - val src = RasterSourceType.deserialize(child.eval(input)) - val tiles = if (applyTiling) src.readAll() else { - src.read(src.extent).right.map(Seq(_)).left.map(Seq(_)) - } - - require(tiles.isLeft, "Multiband tiles are not yet supported") - - tiles.left.get - } - refs.transpose.map(ts ⇒ InternalRow(ts.map(r ⇒ r.tile.toInternalRow): _*)) - } - catch { - case NonFatal(ex) ⇒ - logger.error("Error fetching data for " + sql, ex) - Traversable.empty - } - } -} - - -object RasterSourceToTiles { - def apply(rrs: Column*): Column = apply(true, rrs: _*) - def apply(applyTiling: Boolean, rrs: Column*): Column = - new Column(new RasterSourceToTiles(rrs.map(_.expr), applyTiling)) -} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala deleted file mode 100644 index ef0c901f8..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package astraea.spark.rasterframes.extensions - -import astraea.spark.rasterframes.PairRDDConverter._ -import astraea.spark.rasterframes.StandardColumns._ -import astraea.spark.rasterframes.extensions.Implicits._ -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.{PairRDDConverter, RasterFrame} -import geotrellis.raster.{CellGrid, Tile} -import geotrellis.spark._ -import geotrellis.spark.io._ -import geotrellis.util.MethodExtensions -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession - -/** - * Extension method on `ContextRDD`-shaped RDDs with appropriate context bounds to create a RasterFrame. - * @since 7/18/17 - */ -abstract class SpatialContextRDDMethods[T <: CellGrid](implicit spark: SparkSession) - extends MethodExtensions[RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]] { - import PairRDDConverter._ - - def toRF(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName) - - def toRF(tileColumnName: String)(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrame = { - val df = self.toDataFrame.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata) - val defName = TILE_COLUMN.columnName - df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName)) - .certify - } -} - -/** - * Extension method on `ContextRDD`-shaped [[Tile]] RDDs keyed with [[SpaceTimeKey]], with appropriate context bounds to create a RasterFrame. - * @since 9/11/17 - */ -abstract class SpatioTemporalContextRDDMethods[T <: CellGrid]( - implicit spark: SparkSession) - extends MethodExtensions[RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] { - - def toRF(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName) - - def toRF(tileColumnName: String)(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrame = { - val df = self.toDataFrame - .setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata) - .setTemporalColumnRole(TEMPORAL_KEY_COLUMN) - val defName = TILE_COLUMN.columnName - df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName)) - .certify - } -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/DataFrameMethods.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/DataFrameMethods.scala deleted file mode 100644 index ca38322ac..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/DataFrameMethods.scala +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package astraea.spark.rasterframes.extensions - -import astraea.spark.rasterframes.StandardColumns._ -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.{MetadataKeys, RasterFrame} -import geotrellis.raster.Tile -import geotrellis.spark.io._ -import geotrellis.spark.{SpaceTimeKey, SpatialComponent, SpatialKey, TemporalKey, TileLayerMetadata} -import geotrellis.util.MethodExtensions -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.rf.TileUDT -import org.apache.spark.sql.types.{MetadataBuilder, StructField} -import org.apache.spark.sql.{Column, DataFrame, TypedColumn} -import spray.json.JsonFormat -import astraea.spark.rasterframes.encoders.StandardEncoders._ -import scala.util.Try - -/** - * Extension methods over [[DataFrame]]. - * - * @since 7/18/17 - */ -trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with MetadataKeys { - import Implicits.{WithDataFrameMethods, WithMetadataBuilderMethods, WithMetadataMethods, WithRasterFrameMethods} - - private def selector(column: Column) = (attr: Attribute) ⇒ - attr.name == column.columnName || attr.semanticEquals(column.expr) - - /** Map over the Attribute representation of Columns, modifying the one matching `column` with `op`. */ - private[astraea] def mapColumnAttribute(column: Column, op: Attribute ⇒ Attribute): DF = { - val analyzed = self.queryExecution.analyzed.output - val selects = selector(column) - val attrs = analyzed.map { attr ⇒ - if(selects(attr)) op(attr) else attr - } - self.select(attrs.map(a ⇒ new Column(a)): _*).asInstanceOf[DF] - } - - private[astraea] def addColumnMetadata(column: Column, op: MetadataBuilder ⇒ MetadataBuilder): DF = { - mapColumnAttribute(column, attr ⇒ { - val md = new MetadataBuilder().withMetadata(attr.metadata) - attr.withMetadata(op(md).build) - }) - } - - private[astraea] def fetchMetadataValue[D](column: Column, reader: (Attribute) ⇒ D): Option[D] = { - val analyzed = self.queryExecution.analyzed.output - analyzed.find(selector(column)).map(reader) - } - - private[astraea] - def setSpatialColumnRole[K: SpatialComponent: JsonFormat]( - column: Column, md: TileLayerMetadata[K]): DF = - addColumnMetadata(column, - _.attachContext(md.asColumnMetadata).tagSpatialKey - ) - - private[astraea] - def setTemporalColumnRole(column: Column): DF = - addColumnMetadata(column, _.tagTemporalKey) - - /** Get the role tag the column plays in the RasterFrame, if any. */ - private[astraea] - def getColumnRole(column: Column): Option[String] = - fetchMetadataValue(column, _.metadata.getString(SPATIAL_ROLE_KEY)) - - /** Get the columns that are of type `Tile` */ - def tileColumns: Seq[TypedColumn[Any, Tile]] = - self.schema.fields - .filter(_.dataType.typeName.equalsIgnoreCase(TileUDT.typeName)) - .map(f ⇒ col(f.name).as[Tile]) - - /** Get the columns that are not of type `Tile` */ - def notTileColumns: Seq[Column] = - self.schema.fields - .filterNot(_.dataType.typeName.equalsIgnoreCase(TileUDT.typeName)) - .map(f ⇒ col(f.name)) - - /** Get the spatial column. */ - def spatialKeyColumn: Option[TypedColumn[Any, SpatialKey]] = { - val key = findSpatialKeyField - key - .map(_.name) - .map(col(_).as[SpatialKey]) - } - - /** Get the temporal column, if any. */ - def temporalKeyColumn: Option[TypedColumn[Any, TemporalKey]] = { - val key = findTemporalKeyField - key.map(_.name).map(col(_).as[TemporalKey]) - } - - /** Find the field tagged with the requested `role` */ - private[rasterframes] def findRoleField(role: String): Option[StructField] = - self.schema.fields.find( - f ⇒ - f.metadata.contains(SPATIAL_ROLE_KEY) && - f.metadata.getString(SPATIAL_ROLE_KEY) == role - ) - - /** The spatial key is the first one found with context metadata attached to it. */ - private[rasterframes] def findSpatialKeyField: Option[StructField] = - findRoleField(SPATIAL_KEY_COLUMN.columnName) - - /** The temporal key is the first one found with the temporal tag. */ - private[rasterframes] def findTemporalKeyField: Option[StructField] = - findRoleField(TEMPORAL_KEY_COLUMN.columnName) - - /** Renames all columns such that they start with the given prefix string. - * Useful for preparing dataframes for joins where duplicate names may arise. - */ - def withPrefixedColumnNames(prefix: String): DF = - self.columns.foldLeft(self)((df, c) ⇒ df.withColumnRenamed(c, s"$prefix$c").asInstanceOf[DF]) - - /** Converts this DataFrame to a RasterFrame after ensuring it has: - * - *
    - *
  1. a space or space-time key column - *
  2. one or more tile columns - *
  3. tile layout metadata - *
      - * - * If any of the above are violated, and [[IllegalArgumentException]] is thrown. - * - * @return validated RasterFrame - * @throws IllegalArgumentException when constraints are not met. - */ - @throws[IllegalArgumentException] - def asRF: RasterFrame = { - val potentialRF = certifyRasterframe(self) - - require( - potentialRF.findSpatialKeyField.nonEmpty, - "A RasterFrame requires a column identified as a spatial key" - ) - - require(potentialRF.tileColumns.nonEmpty, "A RasterFrame requires at least one tile column") - - require( - Try(potentialRF.tileLayerMetadata).isSuccess, - "A RasterFrame requires embedded TileLayerMetadata" - ) - - potentialRF - } - - /** - * Convert DataFrame into a RasterFrame - * - * @param spatialKey The column where the spatial key is stored - * @param tlm Metadata describing layout under which tiles were created. Note: no checking is - * performed to ensure metadata, key-space, and tiles are coherent. - * @throws IllegalArgumentException when constraints outlined in `asRF` are not met. - * @return Encoded RasterFrame - */ - @throws[IllegalArgumentException] - def asRF(spatialKey: Column, tlm: TileLayerMetadata[SpatialKey]): RasterFrame = - setSpatialColumnRole(spatialKey, tlm).asRF - - /** - * Convert DataFrame into a RasterFrame - * - * @param spatialKey The column where the spatial key is stored - * @param temporalKey The column tagged under the temporal role - * @param tlm Metadata describing layout under which tiles were created. Note: no checking is - * performed to ensure metadata, key-space, and tiles are coherent. - * @throws IllegalArgumentException when constraints outlined in `asRF` are not met. - * @return Encoded RasterFrame - */ - @throws[IllegalArgumentException] - def asRF(spatialKey: Column, temporalKey: Column, tlm: TileLayerMetadata[SpaceTimeKey]): RasterFrame = - setSpatialColumnRole(spatialKey, tlm) - .setTemporalColumnRole(temporalKey) - .asRF - -// @throws[IllegalArgumentException] -// def asRF(space: LayerSpace): RasterFrame = { -// require(tileColumns.isEmpty, "This method doesn't yet support existing tile columns") -// // We have two use cases to consider: This is already a rasterframe and we need to -// // reproject it. If we have RasterRefs then we reproject those -// val (refFields, otherFields) = self.schema.fields -// .partition(_.dataType.typeName.equalsIgnoreCase(RasterRefUDT.typeName)) -// -// val refCols = refFields.map(f ⇒ self(f.name).as[RasterRef]) -// val otherCols = otherFields.map(f ⇒ self(f.name)) -// -// // Reproject tile into layer space -// val projected = self.select(otherCols :+ ProjectIntoLayer(refCols, space): _*) -// -// // Lastly, convert cell type as desired -// val tileCols = projected.tileColumns.map(c ⇒ convert_cell_type(c, space.cell_type).as(c.columnName)) -// val remCols = projected.notTileColumns -// -// val layer = projected.select(remCols ++ tileCols: _*) -// -// val tlm = space.asTileLayerMetadata -// layer.setSpatialColumnRole(SPATIAL_KEY_COLUMN, tlm).asRF -// } - - /** - * Converts [[DataFrame]] to a RasterFrame if the following constraints are fulfilled: - * - *
        - *
      1. a space or space-time key column - *
      2. one or more tile columns - *
      3. tile layout metadata - *
          - * - * @return Some[RasterFrame] if constraints fulfilled, [[None]] otherwise. - */ - def asRFSafely: Option[RasterFrame] = Try(asRF).toOption - - /** - * Tests for the following conditions on the [[DataFrame]]: - * - *
            - *
          1. a space or space-time key column - *
          2. one or more tile columns - *
          3. tile layout metadata - *
              - * - * @return true if all constraints are fulfilled, false otherwise. - */ - def isRF: Boolean = Try(asRF).isSuccess - - /** Internal method for slapping the RasterFreame seal of approval on a DataFrame. - * Only call if if you are sure it has a spatial key and tile columns and TileLayerMetadata. */ - private[astraea] def certify = certifyRasterframe(self) -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/ref/HttpRangeReader.scala b/core/src/main/scala/astraea/spark/rasterframes/ref/HttpRangeReader.scala deleted file mode 100644 index b0675e8f4..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/ref/HttpRangeReader.scala +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2016 Azavea - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// NB: Copied from -// https://github.com/locationtech/geotrellis/blob/835200c0a6031ddc09fca218ff082cefc9b116c6/spark/src/main/scala/geotrellis/spark/io/http/util/HttpRangeReader.scala -// TODO: Submit PR with changes/fixes. -//package geotrellis.spark.io.http.util - -package astraea.spark.rasterframes.ref - - -import geotrellis.util.{LazyLogging, RangeReader} -import scalaj.http.{Http, HttpResponse} -import java.net.{URI, URL} - -import scala.util.Try - - -/** - * This class extends [[RangeReader]] by reading chunks out of a GeoTiff at the - * specified HTTP location. - * - * @param url: A [[URL]] pointing to the desired GeoTiff. - */ -class HttpRangeReader(url: URL, useHeadRequest: Boolean) extends RangeReader with LazyLogging { - - val request = Http(url.toString) - - lazy val response: HttpResponse[String] = if(useHeadRequest) { - request.method("HEAD").asString - } - else { - request.method("GET").execute { is => "" } - } - - lazy val totalLength: Long = { - /** - * "The Accept-Ranges response HTTP header is a marker used by the server - * to advertise its support of partial requests. The value of this field - * indicates the unit that can be used to define a range." - * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges - */ - require(response.header("Accept-Ranges").contains("bytes"), "Server doesn't support ranged byte reads") - - val contentLength = response - .header("Content-Length") - .flatMap({ cl => Try(cl.toLong).toOption }) match { - case Some(num) => num - case None => -1L - } - - require(contentLength > 0, - "Server didn't provide (required) \"Content-Length\" headers, unable to do range-based read") - - contentLength - } - - def readClippedRange(start: Long, length: Int): Array[Byte] = { - //println("Range read", s"$start-${start + length} ($length bytes)") - - val res = request - .method("GET") - .header("Range", s"bytes=${start}-${start + length}") - .asBytes - - /** - * "If the byte-range-set is unsatisfiable, the server SHOULD return - * a response with a status of 416 (Requested range not satisfiable). - * Otherwise, the server SHOULD return a response with a status of 206 - * (Partial Content) containing the satisfiable ranges of the entity-body." - * https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html - */ - require(res.code != 416, - "Server unable to generate the byte range between ${start} and ${start + length}") - - if (res.code != 206) logger.info("Server responded to range request with HTTP code other than PARTIAL_RESPONSE (206)") - - res.body - } - -} - -/** The companion object of [[HttpRangeReader]] */ -object HttpRangeReader { - - def apply(address: String): HttpRangeReader = apply(new URL(address)) - - def apply(uri: URI): HttpRangeReader = apply(uri.toURL) - - /** - * Returns a new instance of HttpRangeReader. - * - * @param url: A [[URL]] pointing to the desired GeoTiff. - * @return A new instance of HttpRangeReader. - */ - def apply(url: URL): HttpRangeReader = new HttpRangeReader(url, true) - - /** - * Returns a new instance of HttpRangeReader which does not use HEAD - * to determine the totalLength. - * - * @param url: A [[URL]] pointing to the desired GeoTiff. - * @return A new instance of HttpRangeReader. - */ - def withoutHeadRequest(url: URL): HttpRangeReader = new HttpRangeReader(url, false) - - def withoutHeadRequest(address: String): HttpRangeReader = withoutHeadRequest(new URL(address)) - - def withoutHeadRequest(uri: URI): HttpRangeReader = withoutHeadRequest(uri.toURL) -} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/ref/LayerSpace.scala b/core/src/main/scala/astraea/spark/rasterframes/ref/LayerSpace.scala deleted file mode 100644 index 25a9d050c..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/ref/LayerSpace.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.ref - -import astraea.spark.rasterframes.tiles.ProjectedRasterTile -import geotrellis.proj4.CRS -import geotrellis.raster._ -import geotrellis.raster.resample.ResampleMethod -import geotrellis.spark.tiling.LayoutDefinition -import geotrellis.spark.{SpatialKey, _} - - -/** - * NB: This package is only a temporary home for this. - * - * @since 9/5/18 - */ -case class LayerSpace( - crs: CRS, - cellType: CellType, - layout: LayoutDefinition, - resampleMethod: ResampleMethod = ResampleMethod.DEFAULT -) { - - def reproject(dest: CRS): LayerSpace = { - copy( - crs = dest, - layout = layout.copy(extent = layout.extent.reproject(crs, dest)) - ) - } - - def asTileLayerMetadata: TileLayerMetadata[SpatialKey] = { - val bounds = KeyBounds( - SpatialKey(0, 0), - SpatialKey(layout.layoutCols - 1, layout.layoutRows - 1) - ) - TileLayerMetadata(cellType, layout, layout.extent, crs, bounds) - } -} - -object LayerSpace { - - private[rasterframes] - def defaultLayout(prt: ProjectedRasterTile): LayoutDefinition = - LayoutDefinition(prt.extent, TileLayout(1, 1, prt.cols, prt.rows)) - - def from(rs: RasterSource): LayerSpace = new LayerSpace( - rs.crs, rs.cellType, LayoutDefinition(rs.extent, rs.nativeLayout - .getOrElse(TileLayout(1, 1, rs.cols, rs.rows)) - ) - ) - - def from(rr: RasterRef): LayerSpace = new LayerSpace( - rr.crs, rr.cellType, RasterRef.defaultLayout(rr) - ) - - def from(prt: ProjectedRasterTile): LayerSpace = new LayerSpace( - prt.crs, prt.cellType, defaultLayout(prt) - ) - -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/ref/RasterSource.scala b/core/src/main/scala/astraea/spark/rasterframes/ref/RasterSource.scala deleted file mode 100644 index 9dc9bd55e..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/ref/RasterSource.scala +++ /dev/null @@ -1,402 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.ref - -import java.net.URI -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter - -import astraea.spark.rasterframes.NOMINAL_TILE_SIZE -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile -import astraea.spark.rasterframes.tiles.ProjectedRasterTile -import astraea.spark.rasterframes.util.GeoTiffInfoSupport -import com.typesafe.scalalogging.LazyLogging -import geotrellis.proj4.CRS -import geotrellis.raster._ -import geotrellis.raster.io.geotiff.reader.GeoTiffReader -import geotrellis.raster.io.geotiff.{GeoTiffSegmentLayout, MultibandGeoTiff, SinglebandGeoTiff, Tags} -import geotrellis.raster.split.Split -import geotrellis.spark.io.hadoop.HdfsRangeReader -import geotrellis.spark.io.s3.S3Client -import geotrellis.spark.io.s3.util.S3RangeReader -import geotrellis.spark.tiling.LayoutDefinition -import geotrellis.util.{FileRangeReader, RangeReader} -import geotrellis.vector.Extent -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.annotation.Experimental -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.rf.RasterSourceUDT - -import scala.util.Try - -/** - * Abstraction over fetching geospatial raster data. - * - * @since 8/21/18 - */ -@Experimental -sealed trait RasterSource extends ProjectedRasterLike with Serializable { - def crs: CRS - - def extent: Extent - - def timestamp: Option[ZonedDateTime] - - def cellType: CellType - - def bandCount: Int - - def tags: Option[Tags] - - def read(extent: Extent): Either[Raster[Tile], Raster[MultibandTile]] - - /** Reads the given extent as a single multiband raster. */ - def readMultiband(extent: Extent): Raster[MultibandTile] = - read(extent).fold(r => { - r.copy(tile = MultibandTile(r.tile)) - }, identity) - - def readAll(): Either[Seq[Raster[Tile]], Seq[Raster[MultibandTile]]] - def readAllMultiband(): Seq[Raster[MultibandTile]] = - readAll().fold(_.map(r => { - r.copy(tile = MultibandTile(r.tile)) - }), identity) - - def readAllLazy(): Either[Seq[Raster[Tile]], Seq[Raster[MultibandTile]]] = { - val extents = nativeTiling - if (bandCount == 1) { - val rasters = for { - extent ← extents - rr = RasterRef(this, Some(extent)) - tile: Tile = RasterRefTile(rr) - } yield Raster(tile, extent) - Left(rasters) - } - else { - // Need to figure this out. - RasterSource._logger.warn("Lazy reading is not available for multiband images. Performing eager read.") - val rasters = for { - extent ← extents - raster = this.read(extent).right.get - } yield raster - Right(rasters) - } - } - - def nativeLayout: Option[TileLayout] - - def rasterExtent = RasterExtent(extent, cols, rows) - - def cellSize = CellSize(extent, cols, rows) - - def gridExtent = GridExtent(extent, cellSize) - - def tileContext: TileContext = TileContext(extent, crs) - - def nativeTiling: Seq[Extent] = { - nativeLayout.map { tileLayout ⇒ - val layout = LayoutDefinition(extent, tileLayout) - val transform = layout.mapTransform - for { - col ← 0 until tileLayout.layoutCols - row ← 0 until tileLayout.layoutRows - } yield transform(col, row) - } - .getOrElse(Seq(extent)) - } -} - -object RasterSource extends LazyLogging { - implicit def rsEncoder: ExpressionEncoder[RasterSource] = { - RasterSourceUDT // Makes sure UDT is registered first - ExpressionEncoder() - } - - private def _logger = logger - - def apply(source: URI, callback: Option[ReadCallback] = None): RasterSource = - source.getScheme match { - case "http" | "https" ⇒ HttpGeoTiffRasterSource(source, callback) - case "file" ⇒ FileGeoTiffRasterSource(source, callback) - case "hdfs" | "s3n" | "s3a" | "wasb" | "wasbs" ⇒ - // TODO: How can we get the active hadoop configuration - // TODO: without having to pass it through? - val config = () ⇒ new Configuration() - HadoopGeoTiffRasterSource(source, config, callback) - case "s3" ⇒ - val client = () ⇒ S3Client.DEFAULT - S3GeoTiffRasterSource(source, client, callback) - case s ⇒ throw new UnsupportedOperationException(s"Scheme '$s' not supported") - } - - - case class SimpleGeoTiffInfo( - cellType: CellType, - extent: Extent, - rasterExtent: RasterExtent, - crs: CRS, - tags: Tags, - segmentLayout: GeoTiffSegmentLayout, - bandCount: Int, - noDataValue: Option[Double] - ) - - object SimpleGeoTiffInfo { - def apply(info: GeoTiffReader.GeoTiffInfo): SimpleGeoTiffInfo = - SimpleGeoTiffInfo(info.cellType, info.extent, info.rasterExtent, info.crs, info.tags, info.segmentLayout, info.bandCount, info.noDataValue) - } - - // According to https://goo.gl/2z8xx9 the GeoTIFF date format is 'YYYY:MM:DD HH:MM:SS' - private val dateFormat = DateTimeFormatter.ofPattern("yyyy:MM:dd HH:mm:ss") - - trait URIRasterSource { _: RasterSource ⇒ - def source: URI - - abstract override def toString: String = { - s"${getClass.getSimpleName}(${source})" - } - } - - case class InMemoryRasterSource(tile: Tile, extent: Extent, crs: CRS) extends RasterSource { - def this(prt: ProjectedRasterTile) = this(prt, prt.extent, prt.crs) - - override def rows: Int = tile.rows - - override def cols: Int = tile.cols - - override def timestamp: Option[ZonedDateTime] = None - - override def cellType: CellType = tile.cellType - - override def bandCount: Int = 1 - - override def tags: Option[Tags] = None - - override def read(extent: Extent): Either[Raster[Tile], Raster[MultibandTile]] = Left( - Raster(tile.crop(rasterExtent.gridBoundsFor(extent, false)), extent) - ) - - override def nativeLayout: Option[TileLayout] = Some( - TileLayout( - layoutCols = math.ceil(this.cols.toDouble / NOMINAL_TILE_SIZE).toInt, - layoutRows = math.ceil(this.rows.toDouble / NOMINAL_TILE_SIZE).toInt, - tileCols = NOMINAL_TILE_SIZE, - tileRows = NOMINAL_TILE_SIZE) - ) - - def readAll(): Either[Seq[Raster[Tile]], Seq[Raster[MultibandTile]]] = { - Left(Raster(tile, extent).split(nativeLayout.get, Split.Options(false, false)).toSeq) - } - } - - trait RangeReaderRasterSource extends RasterSource with GeoTiffInfoSupport with LazyLogging { - protected def rangeReader: RangeReader - - private def realInfo = - GeoTiffReader.readGeoTiffInfo(rangeReader, streaming = true, withOverviews = false) - - private lazy val tiffInfo = SimpleGeoTiffInfo(realInfo) - - def crs: CRS = tiffInfo.crs - - def extent: Extent = tiffInfo.extent - - def timestamp: Option[ZonedDateTime] = resolveDate - - override def cols: Int = tiffInfo.rasterExtent.cols - - override def rows: Int = tiffInfo.rasterExtent.rows - - def cellType: CellType = tiffInfo.cellType - - def bandCount: Int = tiffInfo.bandCount - - override def tags: Option[Tags] = Option(tiffInfo.tags) - - def nativeLayout: Option[TileLayout] = { - if (tiffInfo.segmentLayout.isTiled) - Some(tiffInfo.segmentLayout.tileLayout) - else None - } - - // TODO: Determine if this is the correct way to handle time. - protected def resolveDate: Option[ZonedDateTime] = { - tiffInfo.tags.headTags - .get(Tags.TIFFTAG_DATETIME) - .flatMap(ds ⇒ Try({ - logger.debug("Parsing header date: " + ds) - ZonedDateTime.parse(ds, dateFormat) - }).toOption) - } - - def read(extent: Extent): Either[Raster[Tile], Raster[MultibandTile]] = { - val info = realInfo - if (bandCount == 1) { - val geoTiffTile = GeoTiffReader.geoTiffSinglebandTile(info) - val gt = new SinglebandGeoTiff( - geoTiffTile, - info.extent, - info.crs, - info.tags, - info.options, - List.empty - ) - Left(gt.crop(extent).raster) - } - else { - val geoTiffTile = GeoTiffReader.geoTiffMultibandTile(info) - val gt = new MultibandGeoTiff( - geoTiffTile, - info.extent, - info.crs, - info.tags, - info.options, - List.empty - ) - Right(gt.crop(extent).raster) - } - } - - def readAll(): Either[Seq[Raster[Tile]], Seq[Raster[MultibandTile]]] = { - val info = realInfo - - // Thanks to @pomadchin for showing us how to do this :-) - val windows = info.segmentLayout.listWindows(NOMINAL_TILE_SIZE) - val re = info.rasterExtent - - if (info.bandCount == 1) { - val geotile = GeoTiffReader.geoTiffSinglebandTile(info) - - val rows = windows.map(gb ⇒ { - val tile = geotile.crop(gb) - val extent = re.extentFor(gb, clamp = false) - Raster(tile, extent) - }) - - Left(rows.toSeq) - } - else { - val geotile = GeoTiffReader.geoTiffMultibandTile(info) - - val rows = windows.map(gb ⇒ { - val tile = geotile.crop(gb) - val extent = re.extentFor(gb, clamp = false) - Raster(tile, extent) - }) - - Right(rows.toSeq) - } - } - } - - case class FileGeoTiffRasterSource(source: URI, callback: Option[ReadCallback]) extends RangeReaderRasterSource - with URIRasterSource with URIRasterSourceDebugString { self ⇒ - @transient - protected lazy val rangeReader = { - val base = FileRangeReader(source.getPath) - // TODO: DRY - callback.map(cb ⇒ ReportingRangeReader(base, cb, self)).getOrElse(base) - } - } - - case class HadoopGeoTiffRasterSource(source: URI, config: () ⇒ Configuration, callback: Option[ReadCallback]) extends RangeReaderRasterSource - with URIRasterSource with URIRasterSourceDebugString { self ⇒ - @transient - protected lazy val rangeReader = { - val base = HdfsRangeReader(new Path(source.getPath), config()) - callback.map(cb ⇒ ReportingRangeReader(base, cb, self)).getOrElse(base) - } - } - - case class S3GeoTiffRasterSource(source: URI, client: () ⇒ S3Client, callback: Option[ReadCallback]) extends RangeReaderRasterSource - with URIRasterSource with URIRasterSourceDebugString { self ⇒ - @transient - protected lazy val rangeReader = { - val base = S3RangeReader(source, client()) - callback.map(cb ⇒ ReportingRangeReader(base, cb, self)).getOrElse(base) - } - } - - case class HttpGeoTiffRasterSource(source: URI, callback: Option[ReadCallback]) extends RangeReaderRasterSource - with URIRasterSource with URIRasterSourceDebugString { self ⇒ - - @transient - protected lazy val rangeReader = { - val base = HttpRangeReader(source) - callback.map(cb ⇒ ReportingRangeReader(base, cb, self)).getOrElse(base) - } - - override protected def resolveDate: Option[ZonedDateTime] = { - super.resolveDate - .orElse { - val hrr = rangeReader match { - case h: HttpRangeReader ⇒ h - case ReportingRangeReader(h: HttpRangeReader, _, _) ⇒ h - } - hrr.response.headers.get("Last-Modified") - .flatMap(_.headOption) - .flatMap(s ⇒ Try(ZonedDateTime.parse(s, DateTimeFormatter.RFC_1123_DATE_TIME)).toOption) - } - } - } - - /** Trait for registering a callback for logging or monitoring range reads. - * NB: the callback will be invoked from within a Spark task, and therefore - * is serialized along with its closure to executors. */ - trait ReadCallback extends Serializable { - def readRange(source: RasterSource, start: Long, length: Int): Unit - } - - private case class ReportingRangeReader(delegate: RangeReader, callback: ReadCallback, parent: RasterSource) extends RangeReader { - override def totalLength: Long = delegate.totalLength - - override protected def readClippedRange(start: Long, length: Int): Array[Byte] = { - callback.readRange(parent, start, length) - delegate.readRange(start, length) - } - } - - trait URIRasterSourceDebugString { - _: RangeReaderRasterSource with URIRasterSource with Product ⇒ - def toDebugString: String = { - val buf = new StringBuilder() - buf.append(productPrefix) - buf.append("(") - buf.append("source=") - buf.append(source.toASCIIString) - buf.append(", size=") - buf.append(size) - buf.append(", dimensions=") - buf.append(dimensions) - buf.append(", crs=") - buf.append(crs) - buf.append(", extent=") - buf.append(extent) - buf.append(", timestamp=") - buf.append(timestamp) - buf.append(")") - buf.toString - } - } - -} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialUDFSubstitutionRules.scala b/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialUDFSubstitutionRules.scala deleted file mode 100644 index c0f985b0b..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialUDFSubstitutionRules.scala +++ /dev/null @@ -1,21 +0,0 @@ -package astraea.spark.rasterframes.rules - -import astraea.spark.rasterframes.expressions.SpatialRelation -import org.apache.spark.sql.catalyst.expressions.ScalaUDF -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.rules.Rule - -/** - * Swaps out spatial relation UDFs for expression forms. - * - * @since 2/19/18 - */ -object SpatialUDFSubstitutionRules extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = { - plan.transform { - case q: LogicalPlan ⇒ q.transformExpressions { - case s: ScalaUDF ⇒ SpatialRelation.fromUDF(s).getOrElse(s) - } - } - } -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/tiles/DelegatingTile.scala b/core/src/main/scala/astraea/spark/rasterframes/tiles/DelegatingTile.scala deleted file mode 100644 index 8f0a910c7..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/tiles/DelegatingTile.scala +++ /dev/null @@ -1,109 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.tiles - -import geotrellis.raster._ - -/** - * A tile that wraps another tile. Originally intended for delayed reading, but useful in other special use cases. - * - * @since 8/22/18 - */ -trait DelegatingTile extends Tile { - protected def delegate: Tile - - def cellType: CellType = - delegate.cellType - - def cols: Int = - delegate.cols - - def rows: Int = - delegate.rows - - def mutable: MutableArrayTile = - delegate.mutable - - def convert(cellType: CellType): Tile = - delegate.convert(cellType) - - override def withNoData(noDataValue: Option[Double]): Tile = - delegate.withNoData(noDataValue) - - def interpretAs(newCellType: CellType): Tile = - delegate.interpretAs(newCellType) - - def get(col: Int, row: Int): Int = - delegate.get(col, row) - - def getDouble(col: Int, row: Int): Double = - delegate.getDouble(col, row) - - def toArrayTile(): ArrayTile = - delegate.toArrayTile() - - def toArray(): Array[Int] = - delegate.toArray() - - def toArrayDouble(): Array[Double] = - delegate.toArrayDouble() - - def toBytes(): Array[Byte] = - delegate.toBytes() - - def foreach(f: Int ⇒ Unit): Unit = - delegate.foreach(f) - - def foreachDouble(f: Double ⇒ Unit): Unit = - delegate.foreachDouble(f) - - def map(f: Int ⇒ Int): Tile = - delegate.map(f) - - def combine(r2: Tile)(f: (Int, Int) ⇒ Int): Tile = (delegate, r2) match { - // Hack until https://github.com/locationtech/geotrellis/issues/2792 - case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combine(r2.toArrayTile())(f) - case _ ⇒ delegate.combine(r2)(f) - } - - def combineDouble(r2: Tile)(f: (Double, Double) ⇒ Double): Tile = (delegate, r2) match { - // Hack until https://github.com/locationtech/geotrellis/issues/2792 - case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combineDouble(r2.toArrayTile())(f) - case _ ⇒ delegate.combineDouble(r2)(f) - } - - def mapDouble(f: Double ⇒ Double): Tile = - delegate.mapDouble(f) - - def foreachIntVisitor(visitor: IntTileVisitor): Unit = - delegate.foreachIntVisitor(visitor) - - def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = - delegate.foreachDoubleVisitor(visitor) - - def mapIntMapper(mapper: IntTileMapper): Tile = - delegate.mapIntMapper(mapper) - - def mapDoubleMapper(mapper: DoubleTileMapper): Tile = - delegate.mapDoubleMapper(mapper) - -} diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/ReadAccumulator.scala b/core/src/main/scala/astraea/spark/rasterframes/util/ReadAccumulator.scala deleted file mode 100644 index f9cdfb48e..000000000 --- a/core/src/main/scala/astraea/spark/rasterframes/util/ReadAccumulator.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.util - -import astraea.spark.rasterframes.ref.RasterSource -import astraea.spark.rasterframes.ref.RasterSource.ReadCallback -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.SparkContext -import org.apache.spark.util.LongAccumulator - -/** - * Support for keeping counts of read operations from RasterSource-s - * - * @since 9/3/18 - */ -case class ReadAccumulator(reads: () ⇒ LongAccumulator, bytes: () ⇒ LongAccumulator) extends ReadCallback { - override def readRange(source: RasterSource, start: Long, length: Int): Unit = { - reads().add(1) - bytes().add(length) - } - override def toString: String = - s"${productPrefix}(reads=${reads().value}, bytes=${bytes().value})" -} - -object ReadAccumulator extends LazyLogging { - def apply(sc: SparkContext, prefix: String): ReadAccumulator = this.synchronized { - val reads = sc.longAccumulator(prefix + ".reads") - val bytes = sc.longAccumulator(prefix + ".bytes") - new ReadAccumulator(() ⇒ reads, () ⇒ bytes) - } -} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala b/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala index edecc44b5..6433ef8d3 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.rf import java.sql.{Date, Timestamp} -import astraea.spark.rasterframes.expressions.SpatialRelation.{Contains, Intersects} -import astraea.spark.rasterframes.rules._ +import org.locationtech.rasterframes.expressions.SpatialRelation.{Contains, Intersects} +import org.locationtech.rasterframes.rules._ import org.apache.spark.sql.catalyst.CatalystTypeConverters.{convertToScala, createToScalaConverter} import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.{Attribute, EmptyRow, Expression, Literal} @@ -30,6 +30,7 @@ import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.{DateType, StringType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import org.locationtech.geomesa.spark.jts.rules.GeometryLiteral +import org.locationtech.rasterframes.rules.{SpatialFilters, TemporalFilters} /** * This is a copy of [[org.apache.spark.sql.execution.datasources.DataSourceStrategy.translateFilter]], modified to add our spatial predicates. diff --git a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala index 2dee38a6e..772bde6fe 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala @@ -23,12 +23,12 @@ package org.apache.spark.sql.rf import java.nio.ByteBuffer -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.ref.RasterSource -import astraea.spark.rasterframes.util.KryoSupport +import org.locationtech.rasterframes.encoders.CatalystSerializer._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.{DataType, UDTRegistration, UserDefinedType, _} +import org.locationtech.rasterframes.encoders.CatalystSerializer +import org.locationtech.rasterframes.ref.RasterSource +import org.locationtech.rasterframes.util.KryoSupport /** * Catalyst representation of a RasterSource. @@ -40,11 +40,11 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] { import RasterSourceUDT._ override def typeName = "rf_rastersource" - override def pyUDT: String = "pyrasterframes.RasterSourceUDT" + override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT" def userClass: Class[RasterSource] = classOf[RasterSource] - override def sqlType: DataType = CatalystSerializer[RasterSource].schema + override def sqlType: DataType = schemaOf[RasterSource] override def serialize(obj: RasterSource): InternalRow = Option(obj) @@ -65,9 +65,12 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] { } } -object RasterSourceUDT extends RasterSourceUDT { +object RasterSourceUDT { UDTRegistration.register(classOf[RasterSource].getName, classOf[RasterSourceUDT].getName) + /** Deserialize a byte array, also used inside the Python API */ + def from(byteArray: Array[Byte]): RasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RasterSource] + implicit val rasterSourceSerializer: CatalystSerializer[RasterSource] = new CatalystSerializer[RasterSource] { override def schema: StructType = StructType(Seq( diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala index 75ac0f7cf..66c0d98a1 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala @@ -20,14 +20,13 @@ */ package org.apache.spark.sql.rf - -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.model.{Cells, TileDataContext} -import astraea.spark.rasterframes.tiles.InternalRowTile import geotrellis.raster._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.{DataType, _} +import org.locationtech.rasterframes.encoders.CatalystSerializer +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.{Cells, TileDataContext} +import org.locationtech.rasterframes.tiles.InternalRowTile /** @@ -40,11 +39,11 @@ class TileUDT extends UserDefinedType[Tile] { import TileUDT._ override def typeName = TileUDT.typeName - override def pyUDT: String = "pyrasterframes.TileUDT" + override def pyUDT: String = "pyrasterframes.rf_types.TileUDT" def userClass: Class[Tile] = classOf[Tile] - def sqlType: StructType = CatalystSerializer[Tile].schema + def sqlType: StructType = schemaOf[Tile] override def serialize(obj: Tile): InternalRow = Option(obj) @@ -57,7 +56,7 @@ class TileUDT extends UserDefinedType[Tile] { case ir: InternalRow ⇒ ir.to[Tile] } .map { - case realIRT: InternalRowTile ⇒ realIRT.toArrayTile() + case realIRT: InternalRowTile ⇒ realIRT.realizedTile case other ⇒ other } .orNull @@ -74,11 +73,10 @@ case object TileUDT { final val typeName: String = "tile" implicit def tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] { - import scala.language.reflectiveCalls override def schema: StructType = StructType(Seq( - StructField("cell_context", CatalystSerializer[TileDataContext].schema, false), - StructField("cell_data", CatalystSerializer[Cells].schema, false) + StructField("cell_context", schemaOf[TileDataContext], false), + StructField("cell_data", schemaOf[Cells], false) )) override def to[R](t: Tile, io: CatalystIO[R]): R = io.create( diff --git a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala index b9eb96981..81418d466 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala @@ -1,18 +1,18 @@ package org.apache.spark.sql.rf -import java.lang.reflect.{Constructor, Method} +import java.lang.reflect.Constructor import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.FunctionRegistry -import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FunctionBuilder, expressionInfo} +import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SQLContext} -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryExpression, Expression, ExpressionDescription, ExpressionInfo, RuntimeReplaceable, ScalaUDF} import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, InvokeLike} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionDescription, ExpressionInfo} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SQLContext} import scala.reflect._ import scala.util.{Failure, Success, Try} diff --git a/core/src/main/scala/org/locationtech/rasterframes/MetadataKeys.scala b/core/src/main/scala/org/locationtech/rasterframes/MetadataKeys.scala new file mode 100644 index 000000000..9eb4000d9 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/MetadataKeys.scala @@ -0,0 +1,34 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes + +/** + * + * @since 2/19/18 + */ +trait MetadataKeys { + /** Key under which ContextRDD metadata is stored. */ + private[rasterframes] val CONTEXT_METADATA_KEY = "_context" + + /** Key under which RasterFrameLayer role a column plays. */ + private[rasterframes] val SPATIAL_ROLE_KEY = "_stRole" +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala b/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala rename to core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala index 83686dbfd..658c0d65d 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala @@ -1,8 +1,29 @@ -package astraea.spark.rasterframes +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.util._ import geotrellis.raster.{MultibandTile, Tile, TileFeature} -import geotrellis.spark.{SpaceTimeKey, SpatialKey, TemporalKey} +import geotrellis.spark.{SpaceTimeKey, SpatialKey} import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.rf.TileUDT @@ -15,7 +36,7 @@ import scala.annotation.implicitNotFound * * @since 4/8/18 */ -@implicitNotFound("An RDD converter is required create a RasterFrame. " + +@implicitNotFound("An RDD converter is required create a RasterFrameLayer. " + "Please provide an implementation of PairRDDConverter[${K}, ${V}].") trait PairRDDConverter[K, V] extends Serializable { val schema: StructType diff --git a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala new file mode 100644 index 000000000..20b11c679 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala @@ -0,0 +1,430 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes +import geotrellis.proj4.CRS +import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp +import geotrellis.raster.{CellType, Tile} +import geotrellis.vector.Extent +import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.functions.{lit, udf} +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.jts.geom.Geometry +import org.locationtech.rasterframes.expressions.TileAssembler +import org.locationtech.rasterframes.expressions.accessors._ +import org.locationtech.rasterframes.expressions.aggregates._ +import org.locationtech.rasterframes.expressions.generators._ +import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.tilestats._ +import org.locationtech.rasterframes.expressions.transformers._ +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.stats._ +import org.locationtech.rasterframes.{functions => F} + +/** + * UDFs for working with Tiles in Spark DataFrames. + * + * @since 4/3/17 + */ +trait RasterFunctions { + import util._ + + // format: off + /** Query the number of (cols, rows) in a Tile. */ + def rf_dimensions(col: Column): TypedColumn[Any, TileDimensions] = GetDimensions(col) + + /** Extracts the bounding box of a geometry as an Extent */ + def st_extent(col: Column): TypedColumn[Any, Extent] = GeometryToExtent(col) + + /** Extracts the bounding box from a RasterSource or ProjectedRasterTile */ + def rf_extent(col: Column): TypedColumn[Any, Extent] = GetExtent(col) + + /** Extracts the CRS from a RasterSource or ProjectedRasterTile */ + def rf_crs(col: Column): TypedColumn[Any, CRS] = GetCRS(col) + + /** Extracts the tile from a ProjectedRasterTile, or passes through a Tile. */ + def rf_tile(col: Column): TypedColumn[Any, Tile] = RealizeTile(col) + + /** Flattens Tile into a double array. */ + def rf_tile_to_array_double(col: Column): TypedColumn[Any, Array[Double]] = + TileToArrayDouble(col) + + /** Flattens Tile into an integer array. */ + def rf_tile_to_array_int(col: Column): TypedColumn[Any, Array[Double]] = + TileToArrayDouble(col) + + @Experimental + /** Convert array in `arrayCol` into a Tile of dimensions `cols` and `rows`*/ + def rf_array_to_tile(arrayCol: Column, cols: Int, rows: Int) = withAlias("rf_array_to_tile", arrayCol)( + udf[Tile, AnyRef](F.arrayToTile(cols, rows)).apply(arrayCol) + ) + + /** Create a Tile from a column of cell data with location indexes and preform cell conversion. */ + def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int, ct: CellType): TypedColumn[Any, Tile] = + rf_convert_cell_type(TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows)), ct).as(cellData.columnName).as[Tile](singlebandTileEncoder) + + /** Create a Tile from a column of cell data with location indexes. */ + def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Column, tileRows: Column): TypedColumn[Any, Tile] = + TileAssembler(columnIndex, rowIndex, cellData, tileCols, tileRows) + + /** Extract the Tile's cell type */ + def rf_cell_type(col: Column): TypedColumn[Any, CellType] = GetCellType(col) + + /** Change the Tile's cell type */ + def rf_convert_cell_type(col: Column, cellType: CellType): TypedColumn[Any, Tile] = + SetCellType(col, cellType) + + /** Change the Tile's cell type */ + def rf_convert_cell_type(col: Column, cellTypeName: String): TypedColumn[Any, Tile] = + SetCellType(col, cellTypeName) + + /** Resample tile to different size based on scalar factor or tile whose dimension to match. Scalar less + * than one will downsample tile; greater than one will upsample. Uses nearest-neighbor. */ + def rf_resample[T: Numeric](tileCol: Column, factorValue: T) = Resample(tileCol, factorValue) + + /** Resample tile to different size based on scalar factor or tile whose dimension to match. Scalar less + * than one will downsample tile; greater than one will upsample. Uses nearest-neighbor. */ + def rf_resample(tileCol: Column, factorCol: Column) = Resample(tileCol, factorCol) + + /** Convert a bounding box structure to a Geometry type. Intented to support multiple schemas. */ + def st_geometry(extent: Column): TypedColumn[Any, Geometry] = ExtentToGeometry(extent) + + /** Extract the extent of a RasterSource or ProjectedRasterTile as a Geometry type. */ + def rf_geometry(raster: Column): TypedColumn[Any, Geometry] = GetGeometry(raster) + + /** Assign a `NoData` value to the Tiles. */ + def rf_with_no_data(col: Column, nodata: Double): TypedColumn[Any, Tile] = withTypedAlias("rf_with_no_data", col)( + udf[Tile, Tile](F.withNoData(nodata)).apply(col) + ) + + /** Compute the full column aggregate floating point histogram. */ + def rf_agg_approx_histogram(col: Column): TypedColumn[Any, CellHistogram] = + HistogramAggregate(col) + + /** Compute the full column aggregate floating point statistics. */ + def rf_agg_stats(col: Column): TypedColumn[Any, CellStatistics] = + CellStatsAggregate(col) + + /** Computes the column aggregate mean. */ + def rf_agg_mean(col: Column) = CellMeanAggregate(col) + + /** Computes the number of non-NoData cells in a column. */ + def rf_agg_data_cells(col: Column): TypedColumn[Any, Long] = CellCountAggregate.DataCells(col) + + /** Computes the number of NoData cells in a column. */ + def rf_agg_no_data_cells(col: Column): TypedColumn[Any, Long] = CellCountAggregate.NoDataCells(col) + + /** Compute the Tile-wise mean */ + def rf_tile_mean(col: Column): TypedColumn[Any, Double] = + TileMean(col) + + /** Compute the Tile-wise sum */ + def rf_tile_sum(col: Column): TypedColumn[Any, Double] = + Sum(col) + + /** Compute the minimum cell value in tile. */ + def rf_tile_min(col: Column): TypedColumn[Any, Double] = + TileMin(col) + + /** Compute the maximum cell value in tile. */ + def rf_tile_max(col: Column): TypedColumn[Any, Double] = + TileMax(col) + + /** Compute TileHistogram of Tile values. */ + def rf_tile_histogram(col: Column): TypedColumn[Any, CellHistogram] = + TileHistogram(col) + + /** Compute statistics of Tile values. */ + def rf_tile_stats(col: Column): TypedColumn[Any, CellStatistics] = + TileStats(col) + + /** Counts the number of non-NoData cells per Tile. */ + def rf_data_cells(tile: Column): TypedColumn[Any, Long] = + DataCells(tile) + + /** Counts the number of NoData cells per Tile. */ + def rf_no_data_cells(tile: Column): TypedColumn[Any, Long] = + NoDataCells(tile) + + /** Returns true if all cells in the tile are NoData.*/ + def rf_is_no_data_tile(tile: Column): TypedColumn[Any, Boolean] = + IsNoDataTile(tile) + + /** Returns true if any cells in the tile are true (non-zero and not NoData). */ + def rf_exists(tile: Column): TypedColumn[Any, Boolean] = Exists(tile) + + /** Returns true if all cells in the tile are true (non-zero and not NoData). */ + def rf_for_all(tile: Column): TypedColumn[Any, Boolean] = ForAll(tile) + + /** Compute cell-local aggregate descriptive statistics for a column of Tiles. */ + def rf_agg_local_stats(col: Column) = + LocalStatsAggregate(col) + + /** Compute the cell-wise/local max operation between Tiles in a column. */ + def rf_agg_local_max(col: Column): TypedColumn[Any, Tile] = LocalTileOpAggregate.LocalMaxUDAF(col) + + /** Compute the cellwise/local min operation between Tiles in a column. */ + def rf_agg_local_min(col: Column): TypedColumn[Any, Tile] = LocalTileOpAggregate.LocalMinUDAF(col) + + /** Compute the cellwise/local mean operation between Tiles in a column. */ + def rf_agg_local_mean(col: Column): TypedColumn[Any, Tile] = LocalMeanAggregate(col) + + /** Compute the cellwise/local count of non-NoData cells for all Tiles in a column. */ + def rf_agg_local_data_cells(col: Column): TypedColumn[Any, Tile] = LocalCountAggregate.LocalDataCellsUDAF(col) + + /** Compute the cellwise/local count of NoData cells for all Tiles in a column. */ + def rf_agg_local_no_data_cells(col: Column): TypedColumn[Any, Tile] = LocalCountAggregate.LocalNoDataCellsUDAF(col) + + /** Cellwise addition between two Tiles or Tile and scalar column. */ + def rf_local_add(left: Column, right: Column): TypedColumn[Any, Tile] = Add(left, right) + + /** Cellwise addition of a scalar value to a tile. */ + def rf_local_add[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Add(tileCol, value) + + /** Cellwise subtraction between two Tiles. */ + def rf_local_subtract(left: Column, right: Column): TypedColumn[Any, Tile] = Subtract(left, right) + + /** Cellwise subtraction of a scalar value from a tile. */ + def rf_local_subtract[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Subtract(tileCol, value) + + /** Cellwise multiplication between two Tiles. */ + def rf_local_multiply(left: Column, right: Column): TypedColumn[Any, Tile] = Multiply(left, right) + + /** Cellwise multiplication of a tile by a scalar value. */ + def rf_local_multiply[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Multiply(tileCol, value) + + /** Cellwise division between two Tiles. */ + def rf_local_divide(left: Column, right: Column): TypedColumn[Any, Tile] = Divide(left, right) + + /** Cellwise division of a tile by a scalar value. */ + def rf_local_divide[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = Divide(tileCol, value) + + /** Perform an arbitrary GeoTrellis `LocalTileBinaryOp` between two Tile columns. */ + def rf_local_algebra(op: LocalTileBinaryOp, left: Column, right: Column): TypedColumn[Any, Tile] = + withTypedAlias(opName(op), left, right)(udf[Tile, Tile, Tile](op.apply).apply(left, right)) + + /** Compute the normalized difference of two tile columns */ + def rf_normalized_difference(left: Column, right: Column) = + NormalizedDifference(left, right) + + /** Constructor for tile column with a single cell value. */ + def rf_make_constant_tile(value: Number, cols: Int, rows: Int, cellType: CellType): TypedColumn[Any, Tile] = + rf_make_constant_tile(value, cols, rows, cellType.name) + + /** Constructor for tile column with a single cell value. */ + def rf_make_constant_tile(value: Number, cols: Int, rows: Int, cellTypeName: String): TypedColumn[Any, Tile] = { + import org.apache.spark.sql.rf.TileUDT.tileSerializer + val constTile = encoders.serialized_literal(F.makeConstantTile(value, cols, rows, cellTypeName)) + withTypedAlias(s"rf_make_constant_tile($value, $cols, $rows, $cellTypeName)")(constTile) + } + + /** Create a column constant tiles of zero */ + def rf_make_zeros_tile(cols: Int, rows: Int, cellType: CellType): TypedColumn[Any, Tile] = + rf_make_zeros_tile(cols, rows, cellType.name) + + /** Create a column constant tiles of zero */ + def rf_make_zeros_tile(cols: Int, rows: Int, cellTypeName: String): TypedColumn[Any, Tile] = { + import org.apache.spark.sql.rf.TileUDT.tileSerializer + val constTile = encoders.serialized_literal(F.tileZeros(cols, rows, cellTypeName)) + withTypedAlias(s"rf_make_zeros_tile($cols, $rows, $cellTypeName)")(constTile) + } + + /** Creates a column of tiles containing all ones */ + def rf_make_ones_tile(cols: Int, rows: Int, cellType: CellType): TypedColumn[Any, Tile] = + rf_make_ones_tile(cols, rows, cellType.name) + + /** Creates a column of tiles containing all ones */ + def rf_make_ones_tile(cols: Int, rows: Int, cellTypeName: String): TypedColumn[Any, Tile] = { + import org.apache.spark.sql.rf.TileUDT.tileSerializer + val constTile = encoders.serialized_literal(F.tileOnes(cols, rows, cellTypeName)) + withTypedAlias(s"rf_make_ones_tile($cols, $rows, $cellTypeName)")(constTile) + } + + /** Where the rf_mask tile contains NODATA, replace values in the source tile with NODATA */ + def rf_mask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] = + Mask.MaskByDefined(sourceTile, maskTile) + + /** Where the `maskTile` equals `maskValue`, replace values in the source tile with `NoData` */ + def rf_mask_by_value(sourceTile: Column, maskTile: Column, maskValue: Column): TypedColumn[Any, Tile] = + Mask.MaskByValue(sourceTile, maskTile, maskValue) + + /** Where the `maskTile` does **not** contain `NoData`, replace values in the source tile with `NoData` */ + def rf_inverse_mask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] = + Mask.InverseMaskByDefined(sourceTile, maskTile) + + /** Where the `maskTile` does **not** equal `maskValue`, replace values in the source tile with `NoData` */ + def rf_inverse_mask_by_value(sourceTile: Column, maskTile: Column, maskValue: Column): TypedColumn[Any, Tile] = + Mask.InverseMaskByValue(sourceTile, maskTile, maskValue) + + /** Create a tile where cells in the grid defined by cols, rows, and bounds are filled with the given value. */ + def rf_rasterize(geometry: Column, bounds: Column, value: Column, cols: Int, rows: Int): TypedColumn[Any, Tile] = + withTypedAlias("rf_rasterize", geometry)( + udf(F.rasterize(_: Geometry, _: Geometry, _: Int, cols, rows)).apply(geometry, bounds, value) + ) + + def rf_rasterize(geometry: Column, bounds: Column, value: Column, cols: Column, rows: Column): TypedColumn[Any, Tile] = + withTypedAlias("rf_rasterize", geometry)( + udf(F.rasterize).apply(geometry, bounds, value, cols, rows) + ) + + /** Reproject a column of geometry from one CRS to another. + * @param sourceGeom Geometry column to reproject + * @param srcCRS Native CRS of `sourceGeom` as a literal + * @param dstCRSCol Destination CRS as a column + */ + def st_reproject(sourceGeom: Column, srcCRS: CRS, dstCRSCol: Column): TypedColumn[Any, Geometry] = + ReprojectGeometry(sourceGeom, srcCRS, dstCRSCol) + + /** Reproject a column of geometry from one CRS to another. + * @param sourceGeom Geometry column to reproject + * @param srcCRSCol Native CRS of `sourceGeom` as a column + * @param dstCRS Destination CRS as a literal + */ + def st_reproject(sourceGeom: Column, srcCRSCol: Column, dstCRS: CRS): TypedColumn[Any, Geometry] = + ReprojectGeometry(sourceGeom, srcCRSCol, dstCRS) + + /** Reproject a column of geometry from one CRS to another. + * @param sourceGeom Geometry column to reproject + * @param srcCRS Native CRS of `sourceGeom` as a literal + * @param dstCRS Destination CRS as a literal + */ + def st_reproject(sourceGeom: Column, srcCRS: CRS, dstCRS: CRS): TypedColumn[Any, Geometry] = + ReprojectGeometry(sourceGeom, srcCRS, dstCRS) + + /** Reproject a column of geometry from one CRS to another. + * @param sourceGeom Geometry column to reproject + * @param srcCRSCol Native CRS of `sourceGeom` as a column + * @param dstCRSCol Destination CRS as a column + */ + def st_reproject(sourceGeom: Column, srcCRSCol: Column, dstCRSCol: Column): TypedColumn[Any, Geometry] = + ReprojectGeometry(sourceGeom, srcCRSCol, dstCRSCol) + + /** Render Tile as ASCII string, for debugging purposes. */ + def rf_render_ascii(col: Column): TypedColumn[Any, String] = + DebugRender.RenderAscii(col) + + /** Render Tile cell values as numeric values, for debugging purposes. */ + def rf_render_matrix(col: Column): TypedColumn[Any, String] = + DebugRender.RenderMatrix(col) + + /** Cellwise less than value comparison between two tiles. */ + def rf_local_less(left: Column, right: Column): TypedColumn[Any, Tile] = + Less(left, right) + + /** Cellwise less than value comparison between a tile and a scalar. */ + def rf_local_less[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + Less(tileCol, value) + + /** Cellwise less than or equal to value comparison between a tile and a scalar. */ + def rf_local_less_equal(left: Column, right: Column): TypedColumn[Any, Tile] = + LessEqual(left, right) + + /** Cellwise less than or equal to value comparison between a tile and a scalar. */ + def rf_local_less_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + LessEqual(tileCol, value) + + /** Cellwise greater than value comparison between two tiles. */ + def rf_local_greater(left: Column, right: Column): TypedColumn[Any, Tile] = + Greater(left, right) + + /** Cellwise greater than value comparison between a tile and a scalar. */ + def rf_local_greater[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + Greater(tileCol, value) + + /** Cellwise greater than or equal to value comparison between two tiles. */ + def rf_local_greater_equal(left: Column, right: Column): TypedColumn[Any, Tile] = + GreaterEqual(left, right) + + /** Cellwise greater than or equal to value comparison between a tile and a scalar. */ + def rf_local_greater_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + GreaterEqual(tileCol, value) + + /** Cellwise equal to value comparison between two tiles. */ + def rf_local_equal(left: Column, right: Column): TypedColumn[Any, Tile] = + Equal(left, right) + + /** Cellwise equal to value comparison between a tile and a scalar. */ + def rf_local_equal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + Equal(tileCol, value) + + /** Cellwise inequality comparison between two tiles. */ + def rf_local_unequal(left: Column, right: Column): TypedColumn[Any, Tile] = + Unequal(left, right) + + /** Cellwise inequality comparison between a tile and a scalar. */ + def rf_local_unequal[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = + Unequal(tileCol, value) + + /** Round cell values to nearest integer without chaning cell type. */ + def rf_round(tileCol: Column): TypedColumn[Any, Tile] = + Round(tileCol) + + /** Compute the absolute value of each cell. */ + def rf_abs(tileCol: Column): TypedColumn[Any, Tile] = + Abs(tileCol) + + /** Take natural logarithm of cell values. */ + def rf_log(tileCol: Column): TypedColumn[Any, Tile] = + Log(tileCol) + + /** Take base 10 logarithm of cell values. */ + def rf_log10(tileCol: Column): TypedColumn[Any, Tile] = + Log10(tileCol) + + /** Take base 2 logarithm of cell values. */ + def rf_log2(tileCol: Column): TypedColumn[Any, Tile] = + Log2(tileCol) + + /** Natural logarithm of one plus cell values. */ + def rf_log1p(tileCol: Column): TypedColumn[Any, Tile] = + Log1p(tileCol) + + /** Exponential of cell values */ + def rf_exp(tileCol: Column): TypedColumn[Any, Tile] = + Exp(tileCol) + + /** Ten to the power of cell values */ + def rf_exp10(tileCol: Column): TypedColumn[Any, Tile] = + Exp10(tileCol) + + /** Two to the power of cell values */ + def rf_exp2(tileCol: Column): TypedColumn[Any, Tile] = + Exp2(tileCol) + + /** Exponential of cell values, less one*/ + def rf_expm1(tileCol: Column): TypedColumn[Any, Tile] = + ExpM1(tileCol) + + /** Return the incoming tile untouched. */ + def rf_identity(tileCol: Column): TypedColumn[Any, Tile] = + Identity(tileCol) + + /** Create a row for each cell in Tile. */ + def rf_explode_tiles(cols: Column*): Column = rf_explode_tiles_sample(1.0, None, cols: _*) + + /** Create a row for each cell in Tile with random sampling and optional seed. */ + def rf_explode_tiles_sample(sampleFraction: Double, seed: Option[Long], cols: Column*): Column = + ExplodeTiles(sampleFraction, seed, cols) + + /** Create a row for each cell in Tile with random sampling (no seed). */ + def rf_explode_tiles_sample(sampleFraction: Double, cols: Column*): Column = + ExplodeTiles(sampleFraction, None, cols) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala b/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala new file mode 100644 index 000000000..2e82ab356 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala @@ -0,0 +1,93 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes + +import java.sql.Timestamp + +import geotrellis.proj4.CRS +import geotrellis.raster.Tile +import geotrellis.spark.{SpatialKey, TemporalKey} +import geotrellis.vector.{Extent, ProjectedExtent} +import org.apache.spark.sql.functions.col +import org.locationtech.jts.geom.{Point => jtsPoint, Polygon => jtsPolygon} +import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +/** + * Constants identifying column in most RasterFrames. + * + * @since 2/19/18 + */ +trait StandardColumns { + /** Default RasterFrameLayer spatial column name. */ + val SPATIAL_KEY_COLUMN = col("spatial_key").as[SpatialKey] + + /** Default RasterFrameLayer temporal column name. */ + val TEMPORAL_KEY_COLUMN = col("temporal_key").as[TemporalKey] + + /** Default RasterFrameLayer timestamp column name */ + val TIMESTAMP_COLUMN = col("timestamp").as[Timestamp] + + /** Default RasterFrameLayer column name for an tile extent as geometry value. */ + // This is a `def` because `PolygonUDT` needs to be initialized first. + def GEOMETRY_COLUMN = col("geometry").as[jtsPolygon] + + /** Default RasterFrameLayer column name for the center coordinates of the tile's bounds. */ + // This is a `def` because `PointUDT` needs to be initialized first. + def CENTER_COLUMN = col("center").as[jtsPoint] + + /** Default Extent column name. */ + def EXTENT_COLUMN = col("extent").as[Extent] + + /** Default ProjectedExtent column name. */ + def PROJECTED_EXTENT_COLUMN = col("proj_extent").as[ProjectedExtent] + + /** Default CRS column name. */ + def CRS_COLUMN = col("crs").as[CRS] + + /** Default RasterFrameLayer column name for an added spatial index. */ + val SPATIAL_INDEX_COLUMN = col("spatial_index").as[Long] + + /** Default RasterFrameLayer tile column name. */ + // This is a `def` because `TileUDT` needs to be initialized first. + def TILE_COLUMN = col("tile").as[Tile] + + /** Default column name for a tile with its CRS and Extent. */ + def PROJECTED_RASTER_COLUMN = col("proj_raster").as[ProjectedRasterTile] + + /** Default RasterFrameLayer `TileFeature.data` column name. */ + val TILE_FEATURE_DATA_COLUMN = col("tile_data") + + /** Default GeoTiff tags column. */ + val METADATA_COLUMN = col("metadata").as[Map[String, String]] + + /** Default column index column for the cells of exploded tiles. */ + val COLUMN_INDEX_COLUMN = col("column_index").as[Int] + + /** Default teil column index column for the cells of exploded tiles. */ + val ROW_INDEX_COLUMN = col("row_index").as[Int] + + /** URI/URL/S3 path to raster. */ + val PATH_COLUMN = col("path").as[String] +} + +object StandardColumns extends StandardColumns diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/CRSEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CRSEncoder.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/CRSEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/CRSEncoder.scala index b6a188d71..39ed8d6f3 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/CRSEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/CRSEncoder.scala @@ -15,13 +15,14 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders - -import astraea.spark.rasterframes.util.CRSParser +package org.locationtech.rasterframes.encoders import geotrellis.proj4.CRS import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.locationtech.rasterframes.model.LazyCRS /** * Custom encoder for GT `CRS`. @@ -33,5 +34,5 @@ object CRSEncoder { "crsProj4", "toProj4String", (CRSEncoder.getClass, "fromString") ) // Not sure why this delegate is necessary, but doGenCode fails without it. - def fromString(str: String): CRS = CRSParser(str) + def fromString(str: String): CRS = LazyCRS(str) } diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializer.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala similarity index 82% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializer.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala index 3f09e1f38..831411557 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializer.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala @@ -19,9 +19,9 @@ * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO +import CatalystSerializer.CatalystIO import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.util.ArrayData @@ -50,6 +50,8 @@ trait CatalystSerializer[T] extends Serializable { object CatalystSerializer extends StandardSerializers { def apply[T: CatalystSerializer]: CatalystSerializer[T] = implicitly + def schemaOf[T: CatalystSerializer]: StructType = apply[T].schema + /** * For some reason `Row` and `InternalRow` share no common base type. Instead of using * structural types (which use reflection), this typeclass is used to normalize access @@ -61,8 +63,8 @@ object CatalystSerializer extends StandardSerializers { def create(values: Any*): R def to[T: CatalystSerializer](t: T): R = CatalystSerializer[T].to(t, this) def toSeq[T: CatalystSerializer](t: Seq[T]): AnyRef - def get[T: CatalystSerializer](d: R, ordinal: Int): T - def getSeq[T: CatalystSerializer](d: R, ordinal: Int): Seq[T] + def get[T >: Null: CatalystSerializer](d: R, ordinal: Int): T + def getSeq[T >: Null: CatalystSerializer](d: R, ordinal: Int): Seq[T] def isNullAt(d: R, ordinal: Int): Boolean def getBoolean(d: R, ordinal: Int): Boolean def getByte(d: R, ordinal: Int): Byte @@ -91,14 +93,14 @@ object CatalystSerializer extends StandardSerializers { override def getString(d: R, ordinal: Int): String = d.getString(ordinal) override def getByteArray(d: R, ordinal: Int): Array[Byte] = d.get(ordinal).asInstanceOf[Array[Byte]] - override def get[T: CatalystSerializer](d: R, ordinal: Int): T = { + override def get[T >: Null: CatalystSerializer](d: R, ordinal: Int): T = { d.getAs[Any](ordinal) match { case r: Row => r.to[T] case o => o.asInstanceOf[T] } } override def toSeq[T: CatalystSerializer](t: Seq[T]): AnyRef = t.map(_.toRow) - override def getSeq[T: CatalystSerializer](d: R, ordinal: Int): Seq[T] = + override def getSeq[T >: Null: CatalystSerializer](d: R, ordinal: Int): Seq[T] = d.getSeq[Row](ordinal).map(_.to[T]) override def encode(str: String): String = str } @@ -118,7 +120,7 @@ object CatalystSerializer extends StandardSerializers { override def getDouble(d: InternalRow, ordinal: Int): Double = d.getDouble(ordinal) override def getString(d: InternalRow, ordinal: Int): String = d.getString(ordinal) override def getByteArray(d: InternalRow, ordinal: Int): Array[Byte] = d.getBinary(ordinal) - override def get[T: CatalystSerializer](d: InternalRow, ordinal: Int): T = { + override def get[T >: Null: CatalystSerializer](d: InternalRow, ordinal: Int): T = { val ser = CatalystSerializer[T] val struct = d.getStruct(ordinal, ser.schema.size) struct.to[T] @@ -127,7 +129,7 @@ object CatalystSerializer extends StandardSerializers { override def toSeq[T: CatalystSerializer](t: Seq[T]): ArrayData = ArrayData.toArrayData(t.map(_.toInternalRow).toArray) - override def getSeq[T: CatalystSerializer](d: InternalRow, ordinal: Int): Seq[T] = { + override def getSeq[T >: Null: CatalystSerializer](d: InternalRow, ordinal: Int): Seq[T] = { val ad = d.getArray(ordinal) val result = Array.ofDim[Any](ad.numElements()).asInstanceOf[Array[T]] ad.foreach( @@ -141,15 +143,20 @@ object CatalystSerializer extends StandardSerializers { } implicit class WithToRow[T: CatalystSerializer](t: T) { - def toInternalRow: InternalRow = CatalystSerializer[T].toInternalRow(t) - def toRow: Row = CatalystSerializer[T].toRow(t) + def toInternalRow: InternalRow = if (t == null) null else CatalystSerializer[T].toInternalRow(t) + def toRow: Row = if (t == null) null else CatalystSerializer[T].toRow(t) } implicit class WithFromInternalRow(val r: InternalRow) extends AnyVal { - def to[T: CatalystSerializer]: T = CatalystSerializer[T].fromInternalRow(r) + def to[T >: Null: CatalystSerializer]: T = if (r == null) null else CatalystSerializer[T].fromInternalRow(r) } implicit class WithFromRow(val r: Row) extends AnyVal { - def to[T: CatalystSerializer]: T = CatalystSerializer[T].fromRow(r) + def to[T >: Null: CatalystSerializer]: T = if (r == null) null else CatalystSerializer[T].fromRow(r) + } + + implicit class WithTypeConformity(val left: DataType) extends AnyVal { + def conformsTo[T >: Null: CatalystSerializer]: Boolean = + org.apache.spark.sql.rf.WithTypeConformity(left).conformsTo(schemaOf[T]) } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializerEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala similarity index 98% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializerEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala index 27e452329..792b74165 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/CatalystSerializerEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala @@ -19,7 +19,8 @@ * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders + import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions._ diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/CellTypeEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/CellTypeEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala index 953c2ed65..ea01d4143 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/CellTypeEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import geotrellis.raster.{CellType, DataType} import org.apache.spark.sql.catalyst.ScalaReflection @@ -26,7 +28,7 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf.VersionShims.InvokeSafely import org.apache.spark.sql.types.{ObjectType, StringType} import org.apache.spark.unsafe.types.UTF8String - +import CatalystSerializer._ import scala.reflect.classTag /** @@ -41,7 +43,7 @@ object CellTypeEncoder { import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects._ val ctType = ScalaReflection.dataTypeFor[DataType] - val schema = CatalystSerializer[CellType].schema + val schema = schemaOf[CellType] val inputObject = BoundReference(0, ctType, nullable = false) val intermediateType = ObjectType(classOf[String]) diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/DelegatingSubfieldEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala similarity index 96% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/DelegatingSubfieldEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala index 9b984b8ad..cf4c2e5ac 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/DelegatingSubfieldEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedAttribute, UnresolvedExtractValue} diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/EnvelopeEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala similarity index 59% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/EnvelopeEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala index 5888a1974..50d66f3e0 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/EnvelopeEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala @@ -1,6 +1,27 @@ -package astraea.spark.rasterframes.encoders +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ -import com.vividsolutions.jts.geom.Envelope +package org.locationtech.rasterframes.encoders + +import org.locationtech.jts.geom.Envelope import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -8,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.objects.NewInstance import org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStruct, Literal} import org.apache.spark.sql.rf.VersionShims.InvokeSafely import org.apache.spark.sql.types._ - +import CatalystSerializer._ import scala.reflect.classTag /** @@ -18,7 +39,7 @@ import scala.reflect.classTag */ object EnvelopeEncoder { - val schema = CatalystSerializer[Envelope].schema + val schema = schemaOf[Envelope] val dataType: DataType = ScalaReflection.dataTypeFor[Envelope] diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/ProjectedExtentEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/ProjectedExtentEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala index 0599f9848..f5b078159 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/ProjectedExtentEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala @@ -15,11 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.vector.ProjectedExtent import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/SparkBasicEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/SparkBasicEncoders.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala index 670d2e217..e2830f7f1 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/SparkBasicEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import org.apache.spark.sql.{Encoder, Encoders} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -38,3 +40,4 @@ private[rasterframes] trait SparkBasicEncoders { implicit val boolEnc: Encoder[Boolean] = Encoders.scalaBoolean } +object SparkBasicEncoders extends SparkBasicEncoders \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala index 625eea1cd..256da58d8 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala @@ -15,28 +15,31 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import java.net.URI import java.sql.Timestamp -import astraea.spark.rasterframes.model._ -import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics} -import com.vividsolutions.jts.geom.Envelope +import org.locationtech.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics} +import org.locationtech.jts.geom.Envelope import geotrellis.proj4.CRS -import geotrellis.raster.{CellSize, CellType, Tile, TileLayout} +import geotrellis.raster.{CellSize, CellType, Raster, Tile, TileLayout} import geotrellis.spark.tiling.LayoutDefinition import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TemporalKey, TemporalProjectedExtent, TileLayerMetadata} import geotrellis.vector.{Extent, ProjectedExtent} +import org.apache.spark.sql.{Encoder, Encoders} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders +import org.locationtech.rasterframes.model.{CellContext, Cells, TileContext, TileDataContext} import scala.reflect.runtime.universe._ /** - * Implicit encoder definitions for RasterFrame types. + * Implicit encoder definitions for RasterFrameLayer types. */ trait StandardEncoders extends SpatialEncoders { object PrimitiveEncoders extends SparkBasicEncoders @@ -48,6 +51,7 @@ trait StandardEncoders extends SpatialEncoders { implicit def stkBoundsEncoder: ExpressionEncoder[KeyBounds[SpaceTimeKey]] = ExpressionEncoder() implicit def extentEncoder: ExpressionEncoder[Extent] = ExpressionEncoder[Extent]() implicit def singlebandTileEncoder: ExpressionEncoder[Tile] = ExpressionEncoder() + implicit def rasterEncoder: ExpressionEncoder[Raster[Tile]] = ExpressionEncoder() implicit def tileLayerMetadataEncoder[K: TypeTag]: ExpressionEncoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder() implicit def crsEncoder: ExpressionEncoder[CRS] = CRSEncoder() implicit def projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ProjectedExtentEncoder() @@ -66,6 +70,7 @@ trait StandardEncoders extends SpatialEncoders { implicit def cellsEncoder: ExpressionEncoder[Cells] = Cells.encoder implicit def tileContextEncoder: ExpressionEncoder[TileContext] = TileContext.encoder implicit def tileDataContextEncoder: ExpressionEncoder[TileDataContext] = TileDataContext.encoder + implicit def extentTilePairEncoder: Encoder[(ProjectedExtent, Tile)] = Encoders.tuple(projectedExtentEncoder, singlebandTileEncoder) } diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardSerializers.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala similarity index 81% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/StandardSerializers.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala index aaff5c534..affe545b8 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardSerializers.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala @@ -19,16 +19,18 @@ * */ -package astraea.spark.rasterframes.encoders -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO -import astraea.spark.rasterframes.util.CRSParser -import com.vividsolutions.jts.geom.Envelope +package org.locationtech.rasterframes.encoders + import geotrellis.proj4.CRS import geotrellis.raster._ import geotrellis.spark._ import geotrellis.spark.tiling.LayoutDefinition import geotrellis.vector._ import org.apache.spark.sql.types._ +import org.locationtech.jts.geom.Envelope +import org.locationtech.rasterframes.TileType +import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _} +import org.locationtech.rasterframes.model.LazyCRS /** Collection of CatalystSerializers for third-party types. */ trait StandardSerializers { @@ -77,7 +79,7 @@ trait StandardSerializers { ) ) override def from[R](row: R, io: CatalystIO[R]): CRS = - CRSParser(io.getString(row, 0)) + LazyCRS(io.getString(row, 0)) } implicit val cellTypeSerializer: CatalystSerializer[CellType] = new CatalystSerializer[CellType] { @@ -93,8 +95,8 @@ trait StandardSerializers { implicit val projectedExtentSerializer: CatalystSerializer[ProjectedExtent] = new CatalystSerializer[ProjectedExtent] { override def schema: StructType = StructType(Seq( - StructField("extent", CatalystSerializer[Extent].schema, false), - StructField("crs", CatalystSerializer[CRS].schema, false) + StructField("extent", schemaOf[Extent], false), + StructField("crs", schemaOf[CRS], false) )) override protected def to[R](t: ProjectedExtent, io: CatalystSerializer.CatalystIO[R]): R = io.create( @@ -187,8 +189,8 @@ trait StandardSerializers { implicit val layoutDefinitionSerializer = new CatalystSerializer[LayoutDefinition] { override def schema: StructType = StructType(Seq( - StructField("extent", CatalystSerializer[Extent].schema, true), - StructField("tileLayout", CatalystSerializer[TileLayout].schema, true) + StructField("extent", schemaOf[Extent], true), + StructField("tileLayout", schemaOf[TileLayout], true) )) override protected def to[R](t: LayoutDefinition, io: CatalystIO[R]): R = io.create( @@ -202,10 +204,10 @@ trait StandardSerializers { ) } - implicit def boundsSerializer[T: CatalystSerializer]: CatalystSerializer[KeyBounds[T]] = new CatalystSerializer[KeyBounds[T]] { + implicit def boundsSerializer[T >: Null: CatalystSerializer]: CatalystSerializer[KeyBounds[T]] = new CatalystSerializer[KeyBounds[T]] { override def schema: StructType = StructType(Seq( - StructField("minKey", CatalystSerializer[T].schema, true), - StructField("maxKey", CatalystSerializer[T].schema, true) + StructField("minKey", schemaOf[T], true), + StructField("maxKey", schemaOf[T], true) )) override protected def to[R](t: KeyBounds[T], io: CatalystIO[R]): R = io.create( @@ -219,13 +221,13 @@ trait StandardSerializers { ) } - def tileLayerMetadataSerializer[T: CatalystSerializer]: CatalystSerializer[TileLayerMetadata[T]] = new CatalystSerializer[TileLayerMetadata[T]] { + def tileLayerMetadataSerializer[T >: Null: CatalystSerializer]: CatalystSerializer[TileLayerMetadata[T]] = new CatalystSerializer[TileLayerMetadata[T]] { override def schema: StructType = StructType(Seq( - StructField("cellType", CatalystSerializer[CellType].schema, false), - StructField("layout", CatalystSerializer[LayoutDefinition].schema, false), - StructField("extent", CatalystSerializer[Extent].schema, false), - StructField("crs", CatalystSerializer[CRS].schema, false), - StructField("bounds", CatalystSerializer[KeyBounds[T]].schema, false) + StructField("cellType", schemaOf[CellType], false), + StructField("layout", schemaOf[LayoutDefinition], false), + StructField("extent", schemaOf[Extent], false), + StructField("crs", schemaOf[CRS], false), + StructField("bounds", schemaOf[KeyBounds[T]], false) )) override protected def to[R](t: TileLayerMetadata[T], io: CatalystIO[R]): R = io.create( @@ -245,6 +247,25 @@ trait StandardSerializers { ) } + implicit def rasterSerializer: CatalystSerializer[Raster[Tile]] = new CatalystSerializer[Raster[Tile]] { + import org.apache.spark.sql.rf.TileUDT.tileSerializer + + override def schema: StructType = StructType(Seq( + StructField("tile", TileType, false), + StructField("extent", schemaOf[Extent], false) + )) + + override protected def to[R](t: Raster[Tile], io: CatalystIO[R]): R = io.create( + io.to(t.tile), + io.to(t.extent) + ) + + override protected def from[R](t: R, io: CatalystIO[R]): Raster[Tile] = Raster( + io.get[Tile](t, 0), + io.get[Extent](t, 1) + ) + } + implicit val spatialKeyTLMSerializer = tileLayerMetadataSerializer[SpatialKey] implicit val spaceTimeKeyTLMSerializer = tileLayerMetadataSerializer[SpaceTimeKey] diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/StringBackedEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala similarity index 96% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/StringBackedEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala index 8dc950b4b..2ec265ccc 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/StringBackedEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/TemporalProjectedExtentEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala similarity index 76% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/TemporalProjectedExtentEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala index 5e44bd7fe..f69f7f160 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/TemporalProjectedExtentEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala @@ -15,20 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders - -import java.time.ZonedDateTime +package org.locationtech.rasterframes.encoders -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.spark.TemporalProjectedExtent -import geotrellis.vector.ProjectedExtent import org.apache.spark.sql.Encoders import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder /** - * Custom encoder for [[ProjectedExtent]]. Necessary because [[geotrellis.proj4.CRS]] within [[ProjectedExtent]] isn't a case class, and [[ZonedDateTime]] doesn't have a natural encoder. + * Custom encoder for `TemporalProjectedExtent`. Necessary because `geotrellis.proj4.CRS` within + * `ProjectedExtent` isn't a case class, and `ZonedDateTime` doesn't have a natural encoder. * * @since 8/2/17 */ diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/TileLayerMetadataEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/TileLayerMetadataEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala index c2ed1bbd4..2f59ea451 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/TileLayerMetadataEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import geotrellis.spark.{KeyBounds, TileLayerMetadata} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -31,7 +33,7 @@ import scala.reflect.runtime.universe._ * @since 7/21/17 */ object TileLayerMetadataEncoder { - import astraea.spark.rasterframes._ + import org.locationtech.rasterframes._ private def fieldEncoders = Seq[(String, ExpressionEncoder[_])]( "cellType" -> cellTypeEncoder, diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/URIEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/URIEncoder.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala index d50cd7803..bbbcf25ea 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/URIEncoder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import java.net.URI diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/package.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala similarity index 81% rename from core/src/main/scala/astraea/spark/rasterframes/encoders/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala index 678bbfcd1..8cb5a6f85 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/encoders/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala @@ -15,10 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes +import org.apache.spark.sql.rf._ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Literal @@ -41,7 +44,12 @@ package object encoders { /** Constructs a catalyst literal expression from anything with a serializer. */ def SerializedLiteral[T >: Null: CatalystSerializer](t: T): Literal = { val ser = CatalystSerializer[T] - Literal.create(ser.toInternalRow(t), ser.schema) + val schema = ser.schema match { + case s if s.conformsTo(TileType.sqlType) => TileType + case s if s.conformsTo(RasterSourceType.sqlType) => RasterSourceType + case s => s + } + Literal.create(ser.toInternalRow(t), schema) } /** Constructs a Dataframe literal column from anything with a serializer. */ diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala similarity index 94% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala index 3fac44c65..bd55345fa 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import com.typesafe.scalalogging.LazyLogging import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.analysis.TypeCheckResult diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala index 02f8fc29e..690658064 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/BinaryRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala @@ -19,9 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions -import astraea.spark.rasterframes.expressions.DynamicExtractors.tileExtractor -import astraea.spark.rasterframes.encoders.CatalystSerializer._ +package org.locationtech.rasterframes.expressions + +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.encoders.CatalystSerializer._ import com.typesafe.scalalogging.LazyLogging import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.analysis.TypeCheckResult diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/DynamicExtractors.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala similarity index 68% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/DynamicExtractors.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala index 1dabc8201..6a7e6e421 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/DynamicExtractors.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala @@ -19,26 +19,28 @@ * */ -package astraea.spark.rasterframes.expressions -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.ref.{ProjectedRasterLike, RasterRef, RasterSource} -import astraea.spark.rasterframes.tiles.ProjectedRasterTile +package org.locationtech.rasterframes.expressions + +import geotrellis.proj4.CRS import geotrellis.raster.{CellGrid, Tile} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.rf.{TileUDT, _} +import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT} import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.{LazyCRS, TileContext} +import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef, RasterSource} +import org.locationtech.rasterframes.tiles.ProjectedRasterTile -private[expressions] +private[rasterframes] object DynamicExtractors { /** Partial function for pulling a tile and its contesxt from an input row. */ lazy val tileExtractor: PartialFunction[DataType, InternalRow => (Tile, Option[TileContext])] = { case _: TileUDT => (row: InternalRow) => (row.to[Tile](TileUDT.tileSerializer), None) - case t if t.conformsTo(CatalystSerializer[ProjectedRasterTile].schema) => + case t if t.conformsTo[ProjectedRasterTile] => (row: InternalRow) => { val prt = row.to[ProjectedRasterTile] (prt, Some(TileContext(prt))) @@ -48,7 +50,7 @@ object DynamicExtractors { lazy val rowTileExtractor: PartialFunction[DataType, Row => (Tile, Option[TileContext])] = { case _: TileUDT => (row: Row) => (row.to[Tile](TileUDT.tileSerializer), None) - case t if t.conformsTo(CatalystSerializer[ProjectedRasterTile].schema) => + case t if t.conformsTo[ProjectedRasterTile] => (row: Row) => { val prt = row.to[ProjectedRasterTile] (prt, Some(TileContext(prt))) @@ -58,21 +60,30 @@ object DynamicExtractors { /** Partial function for pulling a ProjectedRasterLike an input row. */ lazy val projectedRasterLikeExtractor: PartialFunction[DataType, InternalRow ⇒ ProjectedRasterLike] = { case _: RasterSourceUDT ⇒ - (row: InternalRow) ⇒ row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer) - case t if t.conformsTo(CatalystSerializer[ProjectedRasterTile].schema) => + (row: InternalRow) => row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer) + case t if t.conformsTo[ProjectedRasterTile] => (row: InternalRow) => row.to[ProjectedRasterTile] - case t if t.conformsTo(CatalystSerializer[RasterRef].schema) => - (row: InternalRow) ⇒ row.to[RasterRef] + case t if t.conformsTo[RasterRef] => + (row: InternalRow) => row.to[RasterRef] } /** Partial function for pulling a CellGrid from an input row. */ lazy val gridExtractor: PartialFunction[DataType, InternalRow ⇒ CellGrid] = { - case _: TileUDT ⇒ - (row: InternalRow) ⇒ row.to[Tile](TileUDT.tileSerializer) - case _: RasterSourceUDT ⇒ - (row: InternalRow) ⇒ row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer) - case t if t.conformsTo(CatalystSerializer[RasterRef].schema) ⇒ - (row: InternalRow) ⇒ row.to[RasterRef] + case _: TileUDT => + (row: InternalRow) => row.to[Tile](TileUDT.tileSerializer) + case _: RasterSourceUDT => + (row: InternalRow) => row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer) + case t if t.conformsTo[RasterRef] ⇒ + (row: InternalRow) => row.to[RasterRef] + case t if t.conformsTo[ProjectedRasterTile] => + (row: InternalRow) => row.to[ProjectedRasterTile] + } + + lazy val crsExtractor: PartialFunction[DataType, Any => CRS] = { + case _: StringType => + (v: Any) => LazyCRS(v.asInstanceOf[UTF8String].toString) + case t if t.conformsTo[CRS] => + (v: Any) => v.asInstanceOf[InternalRow].to[CRS] } sealed trait TileOrNumberArg @@ -106,9 +117,10 @@ object DynamicExtractors { lazy val intArgExtractor: PartialFunction[DataType, Any => IntegerArg] = { case _: IntegerType | _: ByteType | _: ShortType => { case i: Int => IntegerArg(i) - case b: Byte => IntegerArg(b) + case b: Byte => IntegerArg(b.toInt) case s: Short => IntegerArg(s.toInt) case c: Char => IntegerArg(c.toInt) } } + } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/NullToValue.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala similarity index 95% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/NullToValue.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala index edc52fcf7..8bc98c1e2 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/NullToValue.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala @@ -19,7 +19,8 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnaryExpression diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/OnCellGridExpression.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/OnCellGridExpression.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala index b856ae2be..05d56f7d1 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/OnCellGridExpression.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala @@ -19,9 +19,9 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions -import astraea.spark.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import geotrellis.raster.CellGrid import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/OnTileContextExpression.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/OnTileContextExpression.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala index a8797ae49..78ebd1f5b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/OnTileContextExpression.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala @@ -19,14 +19,14 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions -import astraea.spark.rasterframes.expressions.DynamicExtractors._ -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.locationtech.rasterframes.model.TileContext /** * Implements boilerplate for subtype expressions processing TileUDT (when ProjectedRasterTile), RasterSourceUDT, and diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/SpatialRelation.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/SpatialRelation.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala index e994c8a64..1d6697048 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/SpatialRelation.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala @@ -15,12 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions -import astraea.spark.rasterframes.expressions.SpatialRelation.RelationPredicate -import com.vividsolutions.jts.geom._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.SpatialRelation.RelationPredicate +import geotrellis.vector.Extent +import org.locationtech.jts.geom._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -43,6 +47,9 @@ abstract class SpatialRelation extends BinaryExpression case r: InternalRow ⇒ expr.dataType match { case udt: AbstractGeometryUDT[_] ⇒ udt.deserialize(r) + case dt if dt.conformsTo[Extent] => + val extent = r.to[Extent] + extent.jtsGeom } } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/TileAssembler.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala similarity index 77% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/TileAssembler.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala index c3a32267f..c3fe0e17b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/TileAssembler.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala @@ -19,26 +19,44 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions import java.nio.ByteBuffer -import astraea.spark.rasterframes.expressions.TileAssembler.TileBuffer -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.expressions.TileAssembler.TileBuffer +import org.locationtech.rasterframes.util._ import geotrellis.raster.{DataType => _, _} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate} -import org.apache.spark.sql.catalyst.expressions.{Expression, ImplicitCastInputTypes} -import org.apache.spark.sql.rf.TileUDT +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, ImplicitCastInputTypes} import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import spire.syntax.cfor._ +import org.locationtech.rasterframes.TileType /** * Aggregator for reassembling tiles from from exploded form * * @since 9/24/17 */ +@ExpressionDescription( + usage = "_FUNC_(colIndex, rowIndex, cellValue, tileCols, tileRows) - Assemble tiles from set of column and row indices and cell values.", + arguments = """ + Arguments: + * colIndex - column to place the cellValue in the generated tile + * rowIndex - row to place the cellValue in the generated tile + * cellValue - numeric value to place in the generated tile at colIndex and rowIndex + * tileCols - number of columns in the generated tile + * tileRows - number of rows in the generated tile""", + examples = """ + Examples: + > SELECT _FUNC_(column_index, row_index, cell0, 10, 10) as tile; + ... + > SELECT _FUNC_(column_index, row_index, tile, 10, 10) as tile2 + FROM (SELECT rf_explode_tiles(rf_make_constant_tile(4, 10, 10, 'int8raw')) as tile) + ... + """ +) case class TileAssembler( colIndex: Expression, rowIndex: Expression, @@ -49,13 +67,17 @@ case class TileAssembler( inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[TileBuffer] with ImplicitCastInputTypes { + def this(colIndex: Expression, + rowIndex: Expression, + cellValue: Expression, + tileCols: Expression, + tileRows: Expression) = this(colIndex, rowIndex, cellValue, tileCols, tileRows, 0, 0) + override def children: Seq[Expression] = Seq(colIndex, rowIndex, cellValue, tileCols, tileRows) override def inputTypes = Seq(ShortType, ShortType, DoubleType, ShortType, ShortType) - private val TileType = new TileUDT() - - override def prettyName: String = "assemble_tiles" + override def prettyName: String = "rf_assemble_tiles" override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate = copy(mutableAggBufferOffset = newMutableAggBufferOffset) @@ -118,7 +140,7 @@ case class TileAssembler( val cells = Array.ofDim[Double](length) result.get(cells) val (tileCols, tileRows) = buffer.tileSize - val tile = ArrayTile(cells, tileCols, tileRows) + val tile = ArrayTile(cells, tileCols.toInt, tileRows.toInt) TileType.serialize(tile) } @@ -127,7 +149,7 @@ case class TileAssembler( } object TileAssembler { - import astraea.spark.rasterframes.encoders.StandardEncoders._ + import org.locationtech.rasterframes.encoders.StandardEncoders._ def apply( columnIndex: Column, diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala index 049e6d9a1..46969c226 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions +package org.locationtech.rasterframes.expressions -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import com.typesafe.scalalogging.LazyLogging import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.analysis.TypeCheckResult diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala index a28ae6753..cfea46ebe 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala @@ -19,8 +19,9 @@ * */ -package astraea.spark.rasterframes.expressions -import astraea.spark.rasterframes.expressions.DynamicExtractors.rowTileExtractor +package org.locationtech.rasterframes.expressions + +import org.locationtech.rasterframes.expressions.DynamicExtractors.rowTileExtractor import geotrellis.raster.Tile import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala index f21dc4bb5..8d2b532c8 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/UnaryRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala @@ -19,13 +19,14 @@ * */ -package astraea.spark.rasterframes.expressions -import astraea.spark.rasterframes.expressions.DynamicExtractors._ -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions + +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.locationtech.rasterframes.model.TileContext /** Boilerplate for expressions operating on a single Tile-like . */ trait UnaryRasterOp extends UnaryExpression { diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/ExtractTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala similarity index 72% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/ExtractTile.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala index 7cb7ba3b1..4fc0a0374 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/ExtractTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala @@ -19,25 +19,26 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.tiles.InternalRowTile -import astraea.spark.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.tiles.InternalRowTile +import org.locationtech.rasterframes._ /** Expression to extract at tile from several types that contain tiles.*/ case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def dataType: DataType = new TileUDT() + override def dataType: DataType = TileType - override def nodeName: String = "extract_tile" + override def nodeName: String = "rf_extract_tile" implicit val tileSer = TileUDT.tileSerializer override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile match { case irt: InternalRowTile => irt.mem @@ -47,7 +48,7 @@ case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFall } object ExtractTile { - import astraea.spark.rasterframes.encoders.StandardEncoders.singlebandTileEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder def apply(input: Column): TypedColumn[Any, Tile] = new Column(new ExtractTile(input.expr)).as[Tile] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCRS.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala similarity index 69% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCRS.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala index 1a6d29df0..10efc40b7 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCRS.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala @@ -19,28 +19,34 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.encoders.StandardEncoders.crsEncoder -import astraea.spark.rasterframes.expressions.OnTileContextExpression -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.StandardEncoders.crsEncoder +import org.locationtech.rasterframes.expressions.OnTileContextExpression import geotrellis.proj4.CRS import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext /** * Expression to extract the CRS out of a RasterRef or ProjectedRasterTile column. * * @since 9/9/18 */ +@ExpressionDescription( + usage = "_FUNC_(raster) - Fetches the CRS of a ProjectedRasterTile or RasterSource.", + examples = """ + Examples: + > SELECT _FUNC_(raster); + .... + """) case class GetCRS(child: Expression) extends OnTileContextExpression with CodegenFallback { - override def dataType: DataType = CatalystSerializer[CRS].schema - override def nodeName: String = "crs" + override def dataType: DataType = schemaOf[CRS] + override def nodeName: String = "rf_crs" override def eval(ctx: TileContext): InternalRow = ctx.crs.toInternalRow } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCellType.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala similarity index 77% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCellType.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala index eeb521e4b..869835c5f 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetCellType.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala @@ -19,11 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.OnCellGridExpression +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.OnCellGridExpression import geotrellis.raster.{CellGrid, CellType} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -36,15 +35,15 @@ import org.apache.spark.sql.{Column, TypedColumn} */ case class GetCellType(child: Expression) extends OnCellGridExpression with CodegenFallback { - override def nodeName: String = "cell_type" + override def nodeName: String = "rf_cell_type" - def dataType: DataType = CatalystSerializer[CellType].schema + def dataType: DataType = schemaOf[CellType] /** Implemented by subtypes to process incoming ProjectedRasterLike entity. */ override def eval(cg: CellGrid): Any = cg.cellType.toInternalRow } object GetCellType { - import astraea.spark.rasterframes.encoders.StandardEncoders._ + import org.locationtech.rasterframes.encoders.StandardEncoders._ def apply(col: Column): TypedColumn[Any, CellType] = new Column(new GetCellType(col.expr)).as[CellType] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetDimensions.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala similarity index 59% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetDimensions.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala index 3589dbc1b..dffdfdecb 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetDimensions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala @@ -19,31 +19,36 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.OnCellGridExpression -import astraea.spark.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.OnCellGridExpression import geotrellis.raster.CellGrid import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.model.TileDimensions /** - * Extract a Tile's dimensions + * Extract a raster's dimensions * @since 12/21/17 */ -case class GetDimensions(child: Expression) extends OnCellGridExpression - with CodegenFallback { - override def nodeName: String = "tile_dimensions" +@ExpressionDescription( + usage = "_FUNC_(raster) - Fetches the dimensions (columns & rows) of a Tile, ProjectedRasterTile or RasterSource.", + examples = """ + Examples: + > SELECT _FUNC_(raster); + .... + """) +case class GetDimensions(child: Expression) extends OnCellGridExpression with CodegenFallback { + override def nodeName: String = "rf_dimensions" - def dataType = CatalystSerializer[TileDimensions].schema + def dataType = schemaOf[TileDimensions] override def eval(grid: CellGrid): Any = TileDimensions(grid.cols, grid.rows).toInternalRow } object GetDimensions { - def apply(col: Column): Column = + def apply(col: Column): TypedColumn[Any, TileDimensions] = new Column(new GetDimensions(col.expr)).as[TileDimensions] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetEnvelope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetEnvelope.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala index 551f64eb0..d0c14491b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetEnvelope.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala @@ -19,10 +19,9 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.EnvelopeEncoder -import com.vividsolutions.jts.geom.{Envelope, Geometry} +import org.locationtech.jts.geom.{Envelope, Geometry} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression} @@ -30,13 +29,14 @@ import org.apache.spark.sql.jts.AbstractGeometryUDT import org.apache.spark.sql.rf._ import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.encoders.EnvelopeEncoder /** * Extracts the bounding box (envelope) of arbitrary JTS Geometry. * * @since 2/22/18 */ -@deprecated("Replace usages of this with GeometryToBounds", "11/4/2018") +@deprecated("Replace usages of this with GeometryToExtent", "11/4/2018") case class GetEnvelope(child: Expression) extends UnaryExpression with CodegenFallback { override def nodeName: String = "envelope" @@ -60,7 +60,7 @@ case class GetEnvelope(child: Expression) extends UnaryExpression with CodegenFa } object GetEnvelope { - import astraea.spark.rasterframes.encoders.StandardEncoders._ + import org.locationtech.rasterframes.encoders.StandardEncoders._ def apply(col: Column): TypedColumn[Any, Envelope] = new GetEnvelope(col.expr).asColumn.as[Envelope] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetExtent.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala similarity index 64% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetExtent.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala index c3e664887..2266c69b5 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetExtent.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala @@ -19,28 +19,34 @@ * */ -package astraea.spark.rasterframes.expressions.accessors +package org.locationtech.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.encoders.StandardEncoders.extentEncoder -import astraea.spark.rasterframes.expressions.OnTileContextExpression -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.StandardEncoders.extentEncoder +import org.locationtech.rasterframes.expressions.OnTileContextExpression import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext /** - * Expression to extract the Extent out of a RasterRef or ProjectedRasterTile column. + * Expression to extract the Extent out of a RasterSource or ProjectedRasterTile column. * * @since 9/10/18 */ +@ExpressionDescription( + usage = "_FUNC_(raster) - Fetches the extent (bounding box or envelope) of a ProjectedRasterTile or RasterSource.", + examples = """ + Examples: + > SELECT _FUNC_(raster); + .... + """) case class GetExtent(child: Expression) extends OnTileContextExpression with CodegenFallback { - override def dataType: DataType = CatalystSerializer[Extent].schema - override def nodeName: String = "extent" + override def dataType: DataType = schemaOf[Extent] + override def nodeName: String = "rf_extent" override def eval(ctx: TileContext): InternalRow = ctx.extent.toInternalRow } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala new file mode 100644 index 000000000..7ff3bcfc7 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala @@ -0,0 +1,57 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.accessors + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.jts.JTSTypes +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.jts.geom.Geometry +import org.locationtech.rasterframes.encoders.StandardEncoders.jtsGeometryEncoder +import org.locationtech.rasterframes.expressions.OnTileContextExpression +import org.locationtech.rasterframes.model.TileContext + +/** + * Expression to extract the Extent out of a RasterSource or ProjectedRasterTile column. + * + * @since 9/10/18 + */ +@ExpressionDescription( + usage = "_FUNC_(raster) - Fetches the extent (bounding box or envelope) of a ProjectedRasterTile or RasterSource.", + examples = """ + Examples: + > SELECT _FUNC_(raster); + .... + """) +case class GetGeometry(child: Expression) extends OnTileContextExpression with CodegenFallback { + override def dataType: DataType = JTSTypes.GeometryTypeInstance + override def nodeName: String = "rf_geometry" + override def eval(ctx: TileContext): InternalRow = + JTSTypes.GeometryTypeInstance.serialize(ctx.extent.jtsGeom) +} + +object GetGeometry { + def apply(col: Column): TypedColumn[Any, Geometry] = + new Column(GetGeometry(col.expr)).as[Geometry] +} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetTileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala similarity index 75% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetTileContext.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala index 98b7eb401..6c9a3538a 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/accessors/GetTileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala @@ -19,19 +19,20 @@ * */ -package astraea.spark.rasterframes.expressions.accessors -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions.accessors + +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.UnaryRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def dataType: DataType = CatalystSerializer[TileContext].schema + override def dataType: DataType = schemaOf[TileContext] override def nodeName: String = "get_tile_context" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = @@ -39,7 +40,7 @@ case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenF } object GetTileContext { - import astraea.spark.rasterframes.encoders.StandardEncoders.tileContextEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.tileContextEncoder def apply(input: Column): TypedColumn[Any, TileContext] = new Column(new GetTileContext(input.expr)).as[TileContext] diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala new file mode 100644 index 000000000..d8c9f0ba6 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala @@ -0,0 +1,55 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.accessors + +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.rf.TileUDT +import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext + +@ExpressionDescription( + usage = "_FUNC_(raster) - Extracts the Tile component of a RasterSource, ProjectedRasterTile (or Tile) and ensures the cells are fully fetched.", + examples = """ + Examples: + > SELECT _FUNC_(raster); + .... + """) +case class RealizeTile(child: Expression) extends UnaryRasterOp with CodegenFallback { + override def dataType: DataType = TileType + + override def nodeName: String = "rf_tile" + implicit val tileSer = TileUDT.tileSerializer + + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = + (tile.toArrayTile(): Tile).toInternalRow +} + +object RealizeTile { + def apply(col: Column): TypedColumn[Any, Tile] = + new Column(new RealizeTile(col.expr)).as[Tile] +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellCountAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellCountAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala index 0a4424665..82c2d3f93 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellCountAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.UnaryRasterAggregate -import astraea.spark.rasterframes.expressions.tilestats.{DataCells, NoDataCells} +import org.locationtech.rasterframes.expressions.UnaryRasterAggregate +import org.locationtech.rasterframes.expressions.tilestats.{DataCells, NoDataCells} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, _} import org.apache.spark.sql.types.{LongType, Metadata} @@ -47,8 +47,8 @@ abstract class CellCountAggregate(isData: Boolean) extends UnaryRasterAggregate ) private def CellTest = - if (isData) tileOpAsExpression("data_cells", DataCells.op) - else tileOpAsExpression("no_data_cells", NoDataCells.op) + if (isData) tileOpAsExpression("rf_data_cells", DataCells.op) + else tileOpAsExpression("rf_no_data_cells", NoDataCells.op) val updateExpressions = Seq( If(IsNull(child), count, Add(count, CellTest(child))) @@ -64,7 +64,7 @@ abstract class CellCountAggregate(isData: Boolean) extends UnaryRasterAggregate } object CellCountAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc @ExpressionDescription( usage = "_FUNC_(tile) - Count the total data (non-no-data) cells in a tile column.", @@ -77,7 +77,7 @@ object CellCountAggregate { 92384753""" ) case class DataCells(child: Expression) extends CellCountAggregate(true) { - override def nodeName: String = "agg_data_cells" + override def nodeName: String = "rf_agg_data_cells" } object DataCells { def apply(tile: Column): TypedColumn[Any, Long] = @@ -94,7 +94,7 @@ object CellCountAggregate { 23584""" ) case class NoDataCells(child: Expression) extends CellCountAggregate(false) { - override def nodeName: String = "agg_no_data_cells" + override def nodeName: String = "rf_agg_no_data_cells" } object NoDataCells { def apply(tile: Column): TypedColumn[Any, Long] = diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellMeanAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala similarity index 85% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellMeanAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala index 846f169cb..009a46cf3 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellMeanAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.UnaryRasterAggregate -import astraea.spark.rasterframes.expressions.tilestats.{DataCells, Sum} +import org.locationtech.rasterframes.expressions.UnaryRasterAggregate +import org.locationtech.rasterframes.expressions.tilestats.{DataCells, Sum} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, _} import org.apache.spark.sql.types.{DoubleType, LongType, Metadata} @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} .... """) case class CellMeanAggregate(child: Expression) extends UnaryRasterAggregate { - override def nodeName: String = "agg_mean" + override def nodeName: String = "rf_agg_mean" private lazy val sum = AttributeReference("sum", DoubleType, false, Metadata.empty)() @@ -58,7 +58,7 @@ case class CellMeanAggregate(child: Expression) extends UnaryRasterAggregate { // Cant' figure out why we can't just use the Expression directly // this is necessary to properly handle null rows. For example, // if we use `tilestats.Sum` directly, we get an NPE when the stage is executed. - private val DataCellCounts = tileOpAsExpression("data_cells", DataCells.op) + private val DataCellCounts = tileOpAsExpression("rf_data_cells", DataCells.op) private val SumCells = tileOpAsExpression("sum_cells", Sum.op) override val updateExpressions = Seq( @@ -79,7 +79,7 @@ case class CellMeanAggregate(child: Expression) extends UnaryRasterAggregate { } object CellMeanAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc /** Computes the column aggregate mean. */ def apply(tile: Column): TypedColumn[Any, Double] = new Column(new CellMeanAggregate(tile.expr).toAggregateExpression()).as[Double] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellStatsAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellStatsAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala index cfcde38a5..95c0bd837 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/CellStatsAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala @@ -19,16 +19,16 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.stats.CellStatistics +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.stats.CellStatistics +import org.locationtech.rasterframes.TileType import geotrellis.raster.{Tile, _} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode, Complete} import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.{DataType, _} import org.apache.spark.sql.{Column, Row, TypedColumn} @@ -40,8 +40,6 @@ import org.apache.spark.sql.{Column, Row, TypedColumn} case class CellStatsAggregate() extends UserDefinedAggregateFunction { import CellStatsAggregate.C // TODO: rewrite as a DeclarativeAggregate - private val TileType = new TileUDT() - override def inputSchema: StructType = StructType(StructField("value", TileType) :: Nil) override def dataType: DataType = StructType(Seq( @@ -122,11 +120,11 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction { } object CellStatsAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.cellStatsEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.cellStatsEncoder def apply(col: Column): TypedColumn[Any, CellStatistics] = new Column(new CellStatsAggregateUDAF(col.expr)) - .as(s"agg_stats($col)") // node renaming in class doesn't seem to propogate + .as(s"rf_agg_stats($col)") // node renaming in class doesn't seem to propogate .as[CellStatistics] /** Adapter hack to allow UserDefinedAggregateFunction to be referenced as an expression. */ @@ -147,7 +145,7 @@ object CellStatsAggregate { class CellStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new CellStatsAggregate()), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_stats" + override def nodeName: String = "rf_agg_stats" } object CellStatsAggregateUDAF { def apply(child: Expression): CellStatsAggregateUDAF = new CellStatsAggregateUDAF(child) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/HistogramAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala similarity index 88% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/HistogramAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala index 7920415da..44cc1324b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/HistogramAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala @@ -19,13 +19,13 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates import java.nio.ByteBuffer -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.functions.safeEval -import astraea.spark.rasterframes.stats.CellHistogram +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.functions.safeEval +import org.locationtech.rasterframes.stats.CellHistogram import geotrellis.raster.Tile import geotrellis.raster.histogram.{Histogram, StreamingHistogram} import geotrellis.spark.util.KryoSerializer @@ -33,9 +33,9 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, Row, TypedColumn} +import org.locationtech.rasterframes.TileType /** * Histogram aggregation function for a full column of tiles. @@ -45,7 +45,6 @@ import org.apache.spark.sql.{Column, Row, TypedColumn} case class HistogramAggregate(numBuckets: Int) extends UserDefinedAggregateFunction { def this() = this(StreamingHistogram.DEFAULT_NUM_BUCKETS) // TODO: rewrite as TypedAggregateExpression or similar. - private val TileType = new TileUDT() override def inputSchema: StructType = StructType(StructField("value", TileType) :: Nil) @@ -96,11 +95,11 @@ case class HistogramAggregate(numBuckets: Int) extends UserDefinedAggregateFunct } object HistogramAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.cellHistEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.cellHistEncoder def apply(col: Column): TypedColumn[Any, CellHistogram] = new Column(new HistogramAggregateUDAF(col.expr)) - .as(s"agg_approx_histogram($col)") // node renaming in class doesn't seem to propogate + .as(s"rf_agg_approx_histogram($col)") // node renaming in class doesn't seem to propogate .as[CellHistogram] /** Adapter hack to allow UserDefinedAggregateFunction to be referenced as an expression. */ @@ -117,7 +116,7 @@ object HistogramAggregate { class HistogramAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new HistogramAggregate()), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_approx_histogram" + override def nodeName: String = "rf_agg_approx_histogram" } object HistogramAggregateUDAF { def apply(child: Expression): HistogramAggregateUDAF = new HistogramAggregateUDAF(child) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalCountAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala similarity index 88% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalCountAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala index f427d9ee3..256cd63dd 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalCountAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala @@ -19,19 +19,19 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.functions.safeBinaryOp +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.functions.safeBinaryOp import geotrellis.raster.mapalgebra.local.{Add, Defined, Undefined} import geotrellis.raster.{IntConstantNoDataCellType, Tile} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode, Complete} import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.sql.{Column, Row, TypedColumn} +import org.locationtech.rasterframes.TileType /** * Catalyst aggregate function that counts `NoData` values in a cell-wise fashion. @@ -47,8 +47,6 @@ class LocalCountAggregate(isData: Boolean) extends UserDefinedAggregateFunction private val add = safeBinaryOp(Add.apply(_: Tile, _: Tile)) - private val TileType = new TileUDT() - override def dataType: DataType = TileType override def inputSchema: StructType = StructType(Seq( @@ -83,19 +81,19 @@ class LocalCountAggregate(isData: Boolean) extends UserDefinedAggregateFunction override def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0) } object LocalCountAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.singlebandTileEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder @ExpressionDescription( usage = "_FUNC_(tile) - Compute cell-wise count of non-no-data values." ) class LocalDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(true)), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_local_data_cells" + override def nodeName: String = "rf_agg_local_data_cells" } object LocalDataCellsUDAF { def apply(child: Expression): LocalDataCellsUDAF = new LocalDataCellsUDAF(child) def apply(tile: Column): TypedColumn[Any, Tile] = new Column(new LocalDataCellsUDAF(tile.expr)) - .as(s"agg_local_data_cells($tile)") + .as(s"rf_agg_local_data_cells($tile)") .as[Tile] } @@ -104,13 +102,13 @@ object LocalCountAggregate { ) class LocalNoDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(false)), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_local_no_data_cells" + override def nodeName: String = "rf_agg_local_no_data_cells" } object LocalNoDataCellsUDAF { def apply(child: Expression): LocalNoDataCellsUDAF = new LocalNoDataCellsUDAF(child) def apply(tile: Column): TypedColumn[Any, Tile] = new Column(new LocalNoDataCellsUDAF(tile.expr)) - .as(s"agg_local_no_data_cells($tile)") + .as(s"rf_agg_local_no_data_cells($tile)") .as[Tile] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalMeanAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala similarity index 77% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalMeanAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala index bab1eba20..06741a98c 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalMeanAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala @@ -19,27 +19,26 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.UnaryRasterAggregate -import astraea.spark.rasterframes.expressions.localops.{Add => AddTiles, Divide => DivideTiles} -import astraea.spark.rasterframes.expressions.transformers.SetCellType +import org.locationtech.rasterframes.expressions.UnaryRasterAggregate +import org.locationtech.rasterframes.expressions.localops.{BiasedAdd, Divide => DivideTiles} +import org.locationtech.rasterframes.expressions.transformers.SetCellType import geotrellis.raster.Tile import geotrellis.raster.mapalgebra.local import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionDescription, If, IsNull, Literal} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.TileType @ExpressionDescription( usage = "_FUNC_(tile) - Computes a new tile contining the mean cell values across all tiles in column.", note = "All tiles in the column must be the same size." ) case class LocalMeanAggregate(child: Expression) extends UnaryRasterAggregate { - private val TileType = new TileUDT() override def dataType: DataType = TileType - override def nodeName: String = "agg_local_mean" + override def nodeName: String = "rf_agg_local_mean" private lazy val count = AttributeReference("count", TileType, true)() @@ -60,21 +59,21 @@ case class LocalMeanAggregate(child: Expression) extends UnaryRasterAggregate { override lazy val updateExpressions: Seq[Expression] = Seq( If(IsNull(count), SetCellType(Defined(child), Literal("int32")), - If(IsNull(child), count, AddTiles(count, Defined(child))) + If(IsNull(child), count, BiasedAdd(count, Defined(child))) ), If(IsNull(sum), SetCellType(child, Literal("float64")), - If(IsNull(child), sum, AddTiles(sum, child)) + If(IsNull(child), sum, BiasedAdd(sum, child)) ) ) override val mergeExpressions: Seq[Expression] = Seq( - AddTiles(count.left, count.right), - AddTiles(sum.left, sum.right) + BiasedAdd(count.left, count.right), + BiasedAdd(sum.left, sum.right) ) override lazy val evaluateExpression: Expression = DivideTiles(sum, count) } object LocalMeanAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.singlebandTileEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder def apply(tile: Column): TypedColumn[Any, Tile] = new Column(new LocalMeanAggregate(tile.expr).toAggregateExpression()).as[Tile] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalStatsAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalStatsAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala index 8df684a25..86b360dea 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalStatsAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala @@ -19,21 +19,21 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.functions.safeBinaryOp -import astraea.spark.rasterframes.stats.LocalCellStatistics -import astraea.spark.rasterframes.util.DataBiasedOp.{BiasedAdd, BiasedMax, BiasedMin} +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.functions.safeBinaryOp +import org.locationtech.rasterframes.stats.LocalCellStatistics +import org.locationtech.rasterframes.util.DataBiasedOp.{BiasedAdd, BiasedMax, BiasedMin} import geotrellis.raster.mapalgebra.local._ import geotrellis.raster.{DoubleConstantNoDataCellType, IntConstantNoDataCellType, IntUserDefinedNoDataCellType, Tile} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode, Complete} import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, Row, TypedColumn} +import org.locationtech.rasterframes.TileType /** @@ -44,8 +44,6 @@ import org.apache.spark.sql.{Column, Row, TypedColumn} class LocalStatsAggregate() extends UserDefinedAggregateFunction { import LocalStatsAggregate.C - private val TileType = new TileUDT() - override def inputSchema: StructType = StructType(Seq( StructField("value", TileType, true) )) @@ -149,7 +147,7 @@ object LocalStatsAggregate { def apply(col: Column): TypedColumn[Any, LocalCellStatistics] = new Column(LocalStatsAggregateUDAF(col.expr)) - .as(s"agg_local_stats($col)") + .as(s"rf_agg_local_stats($col)") .as[LocalCellStatistics] /** Adapter hack to allow UserDefinedAggregateFunction to be referenced as an expression. */ @@ -166,7 +164,7 @@ object LocalStatsAggregate { class LocalStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalStatsAggregate()), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_local_stats" + override def nodeName: String = "rf_agg_local_stats" } object LocalStatsAggregateUDAF { def apply(child: Expression): LocalStatsAggregateUDAF = new LocalStatsAggregateUDAF(child) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalTileOpAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalTileOpAggregate.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala index 7a5032176..b739961c1 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/aggstats/LocalTileOpAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala @@ -19,21 +19,20 @@ * */ -package astraea.spark.rasterframes.expressions.aggstats +package org.locationtech.rasterframes.expressions.aggregates -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.functions.safeBinaryOp -import astraea.spark.rasterframes.util.DataBiasedOp.{BiasedMax, BiasedMin} +import org.locationtech.rasterframes.TileType +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.functions.safeBinaryOp +import org.locationtech.rasterframes.util.DataBiasedOp.{BiasedMax, BiasedMin} import geotrellis.raster.Tile -import geotrellis.raster.mapalgebra.local import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp -import org.apache.spark.sql.{Column, Row, TypedColumn} -import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode, Complete} +import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, Row, TypedColumn} /** * Aggregation function for applying a [[LocalTileBinaryOp]] pairwise across all tiles. Assumes Monoid algebra. @@ -44,8 +43,6 @@ class LocalTileOpAggregate(op: LocalTileBinaryOp) extends UserDefinedAggregateFu private val safeOp = safeBinaryOp(op.apply(_: Tile, _: Tile)) - private val TileType = new TileUDT() - override def inputSchema: StructType = StructType(Seq( StructField("value", TileType, true) )) @@ -75,14 +72,14 @@ class LocalTileOpAggregate(op: LocalTileBinaryOp) extends UserDefinedAggregateFu } object LocalTileOpAggregate { - import astraea.spark.rasterframes.encoders.StandardEncoders.singlebandTileEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder @ExpressionDescription( usage = "_FUNC_(tile) - Compute cell-wise minimum value from a tile column." ) class LocalMinUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMin)), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_local_min" + override def nodeName: String = "rf_agg_local_min" } object LocalMinUDAF { def apply(child: Expression): LocalMinUDAF = new LocalMinUDAF(child) @@ -94,7 +91,7 @@ object LocalTileOpAggregate { ) class LocalMaxUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) { def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMax)), Complete, false, NamedExpression.newExprId) - override def nodeName: String = "agg_local_max" + override def nodeName: String = "rf_agg_local_max" } object LocalMaxUDAF { def apply(child: Expression): LocalMaxUDAF = new LocalMaxUDAF(child) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala new file mode 100644 index 000000000..0f1b4727a --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala @@ -0,0 +1,179 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.aggregates + +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.TileDimensions +import geotrellis.proj4.{CRS, Transform} +import geotrellis.raster._ +import geotrellis.raster.reproject.{Reproject, ReprojectRasterExtent} +import geotrellis.spark.tiling.LayoutDefinition +import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata} +import geotrellis.vector.Extent +import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} +import org.apache.spark.sql.types.{DataType, StructField, StructType} +import org.apache.spark.sql.{Column, Row, TypedColumn} + +class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: TileDimensions) extends UserDefinedAggregateFunction { + import ProjectedLayerMetadataAggregate._ + + override def inputSchema: StructType = CatalystSerializer[InputRecord].schema + + override def bufferSchema: StructType = CatalystSerializer[BufferRecord].schema + + override def dataType: DataType = CatalystSerializer[TileLayerMetadata[SpatialKey]].schema + + override def deterministic: Boolean = true + + override def initialize(buffer: MutableAggregationBuffer): Unit = () + + override def update(buffer: MutableAggregationBuffer, input: Row): Unit = { + if(!input.isNullAt(0)) { + val in = input.to[InputRecord] + + if(buffer.isNullAt(0)) { + in.toBufferRecord(destCRS).write(buffer) + } + else { + val br = buffer.to[BufferRecord] + br.merge(in.toBufferRecord(destCRS)).write(buffer) + } + } + } + + override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = { + (buffer1.isNullAt(0), buffer2.isNullAt(0)) match { + case (false, false) ⇒ + val left = buffer1.to[BufferRecord] + val right = buffer2.to[BufferRecord] + left.merge(right).write(buffer1) + case (true, false) ⇒ buffer2.to[BufferRecord].write(buffer1) + case _ ⇒ () + } + } + + override def evaluate(buffer: Row): Any = { + import org.locationtech.rasterframes.encoders.CatalystSerializer._ + val buf = buffer.to[BufferRecord] + + val re = RasterExtent(buf.extent, buf.cellSize) + val layout = LayoutDefinition(re, destDims.cols, destDims.rows) + + val kb = KeyBounds(layout.mapTransform(buf.extent)) + TileLayerMetadata(buf.cellType, layout, buf.extent, destCRS, kb).toRow + } +} + +object ProjectedLayerMetadataAggregate { + import org.locationtech.rasterframes.encoders.StandardEncoders._ + + /** Primary user facing constructor */ + def apply(destCRS: CRS, extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] = + // Ordering must match InputRecord schema + new ProjectedLayerMetadataAggregate(destCRS, TileDimensions(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE))(extent, crs, cellType, tileSize).as[TileLayerMetadata[SpatialKey]] + + def apply(destCRS: CRS, destDims: TileDimensions, extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] = + // Ordering must match InputRecord schema + new ProjectedLayerMetadataAggregate(destCRS, destDims)(extent, crs, cellType, tileSize).as[TileLayerMetadata[SpatialKey]] + + private[expressions] + case class InputRecord(extent: Extent, crs: CRS, cellType: CellType, tileSize: TileDimensions) { + def toBufferRecord(destCRS: CRS): BufferRecord = { + val transform = Transform(crs, destCRS) + + val re = ReprojectRasterExtent( + RasterExtent(extent, tileSize.cols, tileSize.rows), + transform, Reproject.Options.DEFAULT + ) + + BufferRecord( + re.extent, + cellType, + re.cellSize + ) + } + } + + private[expressions] + object InputRecord { + implicit val serializer: CatalystSerializer[InputRecord] = new CatalystSerializer[InputRecord]{ + override def schema: StructType = StructType(Seq( + StructField("extent", CatalystSerializer[Extent].schema, false), + StructField("crs", CatalystSerializer[CRS].schema, false), + StructField("cellType", CatalystSerializer[CellType].schema, false), + StructField("tileSize", CatalystSerializer[TileDimensions].schema, false) + )) + + override protected def to[R](t: InputRecord, io: CatalystIO[R]): R = + throw new IllegalStateException("InputRecord is input only.") + + override protected def from[R](t: R, io: CatalystIO[R]): InputRecord = InputRecord( + io.get[Extent](t, 0), + io.get[CRS](t, 1), + io.get[CellType](t, 2), + io.get[TileDimensions](t, 3) + ) + } + } + + private[expressions] + case class BufferRecord(extent: Extent, cellType: CellType, cellSize: CellSize) { + def merge(that: BufferRecord): BufferRecord = { + val ext = this.extent.combine(that.extent) + val ct = this.cellType.union(that.cellType) + val cs = if (this.cellSize.resolution < that.cellSize.resolution) this.cellSize else that.cellSize + BufferRecord(ext, ct, cs) + } + + def write(buffer: MutableAggregationBuffer): Unit = { + val encoded = (this).toRow + for(i <- 0 until encoded.size) { + buffer(i) = encoded(i) + } + } + } + + private[expressions] + object BufferRecord { + implicit val serializer: CatalystSerializer[BufferRecord] = new CatalystSerializer[BufferRecord] { + override def schema: StructType = StructType(Seq( + StructField("extent", CatalystSerializer[Extent].schema, true), + StructField("cellType", CatalystSerializer[CellType].schema, true), + StructField("cellSize", CatalystSerializer[CellSize].schema, true) + )) + + override protected def to[R](t: BufferRecord, io: CatalystIO[R]): R = io.create( + io.to(t.extent), + io.to(t.cellType), + io.to(t.cellSize) + ) + + override protected def from[R](t: R, io: CatalystIO[R]): BufferRecord = BufferRecord( + io.get[Extent](t, 0), + io.get[CellType](t, 1), + io.get[CellSize](t, 2) + ) + } + } +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala new file mode 100644 index 000000000..e1b11ae3b --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala @@ -0,0 +1,109 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.aggregates + +import geotrellis.proj4.CRS +import geotrellis.raster.reproject.Reproject +import geotrellis.raster.resample.ResampleMethod +import geotrellis.raster.{ArrayTile, CellType, Raster, Tile} +import geotrellis.spark.TileLayerMetadata +import geotrellis.vector.Extent +import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction} +import org.apache.spark.sql.types.{DataType, StructField, StructType} +import org.apache.spark.sql.{Column, Row, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate.ProjectedRasterDefinition + +/** + * Aggregation function for creating a single `geotrellis.raster.Raster[Tile]` from + * `Tile`, `CRS` and `Extent` columns. + * @param prd aggregation settings + */ +class TileRasterizerAggregate(prd: ProjectedRasterDefinition) extends UserDefinedAggregateFunction { + + val projOpts = Reproject.Options.DEFAULT.copy(method = prd.sampler) + + override def deterministic: Boolean = true + + override def inputSchema: StructType = StructType(Seq( + StructField("crs", schemaOf[CRS], false), + StructField("extent", schemaOf[Extent], false), + StructField("tile", TileType) + )) + + override def bufferSchema: StructType = StructType(Seq( + StructField("tile_buffer", TileType) + )) + + override def dataType: DataType = schemaOf[Raster[Tile]] + + override def initialize(buffer: MutableAggregationBuffer): Unit = { + buffer(0) = ArrayTile.empty(prd.cellType, prd.totalCols, prd.totalRows) + } + + override def update(buffer: MutableAggregationBuffer, input: Row): Unit = { + val crs = input.getAs[Row](0).to[CRS] + val extent = input.getAs[Row](1).to[Extent] + + val localExtent = extent.reproject(crs, prd.crs) + + if (prd.extent.intersects(localExtent)) { + val localTile = input.getAs[Tile](2).reproject(extent, crs, prd.crs, projOpts) + val bt = buffer.getAs[Tile](0) + val merged = bt.merge(prd.extent, localExtent, localTile.tile, prd.sampler) + buffer(0) = merged + } + } + + override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = { + val leftTile = buffer1.getAs[Tile](0) + val rightTile = buffer2.getAs[Tile](0) + buffer1(0) = leftTile.merge(rightTile) + } + + override def evaluate(buffer: Row): Raster[Tile] = { + val t = buffer.getAs[Tile](0) + Raster(t, prd.extent) + } +} + +object TileRasterizerAggregate { + val nodeName = "rf_tile_rasterizer_aggregate" + /** Convenience grouping of parameters needed for running aggregate. */ + case class ProjectedRasterDefinition(totalCols: Int, totalRows: Int, cellType: CellType, crs: CRS, extent: Extent, sampler: ResampleMethod = ResampleMethod.DEFAULT) + + object ProjectedRasterDefinition { + def apply(tlm: TileLayerMetadata[_]): ProjectedRasterDefinition = apply(tlm, ResampleMethod.DEFAULT) + + def apply(tlm: TileLayerMetadata[_], sampler: ResampleMethod): ProjectedRasterDefinition = { + // Try to determine the actual dimensions of our data coverage + val actualSize = tlm.layout.toRasterExtent().gridBoundsFor(tlm.extent) // <--- Do we have the math right here? + val cols = actualSize.width + val rows = actualSize.height + new ProjectedRasterDefinition(cols, rows, tlm.cellType, tlm.crs, tlm.extent, sampler) + } +} + + def apply(prd: ProjectedRasterDefinition, crsCol: Column, extentCol: Column, tileCol: Column): TypedColumn[Any, Raster[Tile]] = + new TileRasterizerAggregate(prd)(crsCol, extentCol, tileCol).as(nodeName).as[Raster[Tile]] +} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/generators/ExplodeTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/generators/ExplodeTiles.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala index e39ca1814..bd2a4689a 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/generators/ExplodeTiles.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala @@ -19,11 +19,11 @@ * */ -package astraea.spark.rasterframes.expressions.generators +package org.locationtech.rasterframes.expressions.generators -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.util._ import geotrellis.raster._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.InternalRow @@ -43,7 +43,7 @@ case class ExplodeTiles( extends Expression with Generator with CodegenFallback { def this(children: Seq[Expression]) = this(1.0, None, children) - override def nodeName: String = "explode_tiles" + override def nodeName: String = "rf_explode_tiles" override def elementSchema: StructType = { val names = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala new file mode 100644 index 000000000..68c7209e5 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala @@ -0,0 +1,93 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.generators + +import com.typesafe.scalalogging.LazyLogging +import geotrellis.vector.Extent +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.{DataType, StructField, StructType} +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.ref.{RasterRef, RasterSource} +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.RasterSourceType + +import scala.util.Try +import scala.util.control.NonFatal + +/** + * Accepts RasterSource and generates one or more RasterRef instances representing + * + * @since 9/6/18 + */ +case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[TileDimensions] = None) extends Expression + with Generator with CodegenFallback with ExpectsInputTypes with LazyLogging { + + override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) + override def nodeName: String = "rf_raster_source_to_raster_ref" + + override def elementSchema: StructType = StructType(for { + child <- children + basename = child.name + "_ref" + name <- bandNames(basename, bandIndexes) + } yield StructField(name, schemaOf[RasterRef], true)) + + private def band2ref(src: RasterSource, e: Option[Extent])(b: Int): RasterRef = + if (b < src.bandCount) RasterRef(src, b, e) else null + + override def eval(input: InternalRow): TraversableOnce[InternalRow] = { + try { + val refs = children.map { child ⇒ + val src = RasterSourceType.deserialize(child.eval(input)) + subtileDims.map(dims => + src + .layoutExtents(dims) + .map(e ⇒ bandIndexes.map(band2ref(src, Some(e)))) + ) + .getOrElse(Seq(bandIndexes.map(band2ref(src, None)))) + } + refs.transpose.map(ts ⇒ InternalRow(ts.flatMap(_.map(_.toInternalRow)): _*)) + } + catch { + case NonFatal(ex) ⇒ + val payload = Try(children.map(c => RasterSourceType.deserialize(c.eval(input)))).toOption.toSeq.flatten + logger.error("Error fetching data for one of: " + payload.mkString(", "), ex) + Traversable.empty + } + } +} + +object RasterSourceToRasterRefs { + def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), rrs: _*) + def apply(subtileDims: Option[TileDimensions], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, RasterRef] = + new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[RasterRef] + + private[rasterframes] def bandNames(basename: String, bandIndexes: Seq[Int]): Seq[String] = bandIndexes match { + case Seq() => Seq.empty + case Seq(0) => Seq(basename) + case s => s.map(n => basename + "_b" + n) + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala new file mode 100644 index 000000000..32b3f4b11 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala @@ -0,0 +1,88 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.generators + +import com.typesafe.scalalogging.LazyLogging +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.{DataType, StructField, StructType} +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.RasterSourceType + +import scala.util.Try +import scala.util.control.NonFatal + +/** + * Accepts RasterRef and generates one or more RasterRef instances representing the + * native internal sub-tiling, if any (and requested). + * + * @since 9/6/18 + */ +case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[TileDimensions] = None) extends Expression + with Generator with CodegenFallback with ExpectsInputTypes with LazyLogging { + + override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) + override def nodeName: String = "rf_raster_source_to_tiles" + + override def elementSchema: StructType = StructType(for { + child <- children + basename = child.name + name <- bandNames(basename, bandIndexes) + } yield StructField(name, schemaOf[ProjectedRasterTile], true)) + + override def eval(input: InternalRow): TraversableOnce[InternalRow] = { + try { + val tiles = children.map { child ⇒ + val src = RasterSourceType.deserialize(child.eval(input)) + val maxBands = src.bandCount + val allowedBands = bandIndexes.filter(_ < maxBands) + src.readAll(subtileDims.getOrElse(rasterframes.NOMINAL_TILE_DIMS), allowedBands) + .map(r => bandIndexes.map { + case i if i < maxBands => ProjectedRasterTile(r.tile.band(i), r.extent, src.crs) + case _ => null + }) + } + tiles.transpose.map(ts ⇒ InternalRow(ts.flatMap(_.map(_.toInternalRow)): _*)) + } + catch { + case NonFatal(ex) ⇒ + val payload = Try(children.map(c => RasterSourceType.deserialize(c.eval(input)))).toOption.toSeq.flatten + logger.error("Error fetching data for one of: " + payload.mkString(", "), ex) + Traversable.empty + } + } +} + +object RasterSourceToTiles { + def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*) + def apply(subtileDims: Option[TileDimensions], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = + new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile] +} + + diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala new file mode 100644 index 000000000..0fe6cac87 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala @@ -0,0 +1,50 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.localops + +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} + +@ExpressionDescription( + usage = "_FUNC_(tile) - Compute the absolute value of each cell.", + arguments = """ + Arguments: + * tile - tile column to apply abs""", + examples = """ + Examples: + > SELECT _FUNC_(tile); + ...""" +) +case class Abs(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { + override def nodeName: String = "rf_abs" + override def na: Any = null + override protected def op(t: Tile): Tile = t.localAbs() +} + +object Abs { + def apply(tile: Column): TypedColumn[Any, Tile] = + new Column(Abs(tile.expr)).as[Tile] +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Add.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala similarity index 76% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Add.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala index d7f1a7867..b7a3c8946 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Add.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala @@ -19,19 +19,17 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.DynamicExtractors.tileExtractor -import astraea.spark.rasterframes.expressions.{BinaryLocalRasterOp, DynamicExtractors} -import astraea.spark.rasterframes.util.DataBiasedOp.BiasedAdd import geotrellis.raster.Tile -import org.apache.spark.sql.rf._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise addition between two tiles or a tile and a scalar.", @@ -48,10 +46,10 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class Add(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_add" - override protected def op(left: Tile, right: Tile): Tile = BiasedAdd(left, right) - override protected def op(left: Tile, right: Double): Tile = BiasedAdd(left, right) - override protected def op(left: Tile, right: Int): Tile = BiasedAdd(left, right) + override val nodeName: String = "rf_local_add" + override protected def op(left: Tile, right: Tile): Tile = left.localAdd(right) + override protected def op(left: Tile, right: Double): Tile = left.localAdd(right) + override protected def op(left: Tile, right: Int): Tile = left.localAdd(right) override def eval(input: InternalRow): Any = { if(input == null) null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala new file mode 100644 index 000000000..10bd82e62 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala @@ -0,0 +1,74 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.localops +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.util.DataBiasedOp + +@ExpressionDescription( + usage = "_FUNC_(tile, rhs) - Performs cell-wise addition between two tiles or a tile and a scalar. " + + "Unlike a regular 'add', this considers ` + = .", + arguments = """ + Arguments: + * tile - left-hand-side tile + * rhs - a tile or scalar value to add to each cell""", + examples = """ + Examples: + > SELECT _FUNC_(tile, 1.5); + ... + > SELECT _FUNC_(tile1, tile2); + ...""" +) +case class BiasedAdd(left: Expression, right: Expression) extends BinaryLocalRasterOp + with CodegenFallback { + override val nodeName: String = "rf_local_biased_add" + override protected def op(left: Tile, right: Tile): Tile = DataBiasedOp.BiasedAdd(left, right) + override protected def op(left: Tile, right: Double): Tile = DataBiasedOp.BiasedAdd(left, right) + override protected def op(left: Tile, right: Int): Tile = DataBiasedOp.BiasedAdd(left, right) + + override def eval(input: InternalRow): Any = { + if(input == null) null + else { + val l = left.eval(input) + val r = right.eval(input) + if (l == null && r == null) null + else if (l == null) r + else if (r == null && tileExtractor.isDefinedAt(right.dataType)) l + else if (r == null) null + else nullSafeEval(l, r) + } + } +} +object BiasedAdd { + def apply(left: Column, right: Column): TypedColumn[Any, Tile] = + new Column(BiasedAdd(left.expr, right.expr)).as[Tile] + + def apply[N: Numeric](tile: Column, value: N): TypedColumn[Any, Tile] = + new Column(BiasedAdd(tile.expr, lit(value).expr)).as[Tile] +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Divide.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Divide.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala index 37aa4ab6c..2b5f7d112 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Divide.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -43,7 +43,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Divide(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_divide" + override val nodeName: String = "rf_local_divide" override protected def op(left: Tile, right: Tile): Tile = left.localDivide(right) override protected def op(left: Tile, right: Double): Tile = left.localDivide(right) override protected def op(left: Tile, right: Int): Tile = left.localDivide(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Equal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Equal.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala index 610b8beff..a9e809b47 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Equal.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Equal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_equal" + override val nodeName: String = "rf_local_equal" override protected def op(left: Tile, right: Tile): Tile = left.localEqual(right) override protected def op(left: Tile, right: Double): Tile = left.localEqual(right) override protected def op(left: Tile, right: Int): Tile = left.localEqual(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Exp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Exp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala index 40d34ee06..ca9905e29 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Exp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Exp(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "exp" + override val nodeName: String = "rf_exp" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E) @@ -63,7 +63,7 @@ object Exp { ...""" ) case class Exp10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "log10" + override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(10.0) @@ -84,7 +84,7 @@ object Exp10 { ...""" ) case class Exp2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "exp2" + override val nodeName: String = "rf_exp2" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0) @@ -105,7 +105,7 @@ object Exp2{ ...""" ) case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "expm1" + override val nodeName: String = "rf_expm1" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Greater.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Greater.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala index f78022972..ad9b8bf03 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Greater.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala @@ -18,10 +18,10 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -40,7 +40,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Greater(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_greater" + override val nodeName: String = "rf_local_greater" override protected def op(left: Tile, right: Tile): Tile = left.localGreater(right) override protected def op(left: Tile, right: Double): Tile = left.localGreater(right) override protected def op(left: Tile, right: Int): Tile = left.localGreater(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/GreaterEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/GreaterEqual.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala index bf43ceca5..725898ca5 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/GreaterEqual.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class GreaterEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_greater_equal" + override val nodeName: String = "rf_local_greater_equal" override protected def op(left: Tile, right: Tile): Tile = left.localGreaterOrEqual(right) override protected def op(left: Tile, right: Double): Tile = left.localGreaterOrEqual(right) override protected def op(left: Tile, right: Int): Tile = left.localGreaterOrEqual(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala new file mode 100644 index 000000000..60e607f8b --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala @@ -0,0 +1,50 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.localops + +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} + +@ExpressionDescription( + usage = "_FUNC_(tile) - Return the given tile or projected raster unchanged. Useful in debugging round-trip serialization across various language and memory boundaries.", + arguments = """ + Arguments: + * tile - tile column to pass through""", + examples = """ + Examples: + > SELECT _FUNC_(tile); + ...""" +) +case class Identity(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { + override def nodeName: String = "rf_identity" + override def na: Any = null + override protected def op(t: Tile): Tile = t +} + +object Identity { + def apply(tile: Column): TypedColumn[Any, Tile] = + new Column(Identity(tile.expr)).as[Tile] +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Less.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Less.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala index 4f8d4ad7b..a80d628f7 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Less.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala @@ -18,10 +18,10 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -40,7 +40,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Less(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_less" + override val nodeName: String = "rf_local_less" override protected def op(left: Tile, right: Tile): Tile = left.localLess(right) override protected def op(left: Tile, right: Double): Tile = left.localLess(right) override protected def op(left: Tile, right: Int): Tile = left.localLess(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/LessEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/LessEqual.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala index 983ac7c0d..b9361610b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/LessEqual.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class LessEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_less_equal" + override val nodeName: String = "rf_local_less_equal" override protected def op(left: Tile, right: Tile): Tile = left.localLessOrEqual(right) override protected def op(left: Tile, right: Double): Tile = left.localLessOrEqual(right) override protected def op(left: Tile, right: Int): Tile = left.localLessOrEqual(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Log.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Log.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala index e2da78ce1..8e8d8a011 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Log.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -63,7 +63,7 @@ object Log { ...""" ) case class Log10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "log10" + override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localLog10() @@ -84,7 +84,7 @@ object Log10 { ...""" ) case class Log2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "log2" + override val nodeName: String = "rf_log2" override protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0) @@ -105,7 +105,7 @@ object Log2{ ...""" ) case class Log1p(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "log1p" + override val nodeName: String = "rf_log1p" override protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog() diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Multiply.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Multiply.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala index 7ed7c76b8..784771906 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Multiply.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -43,7 +43,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Multiply(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_multiply" + override val nodeName: String = "rf_local_multiply" override protected def op(left: Tile, right: Tile): Tile = left.localMultiply(right) override protected def op(left: Tile, right: Double): Tile = left.localMultiply(right) override protected def op(left: Tile, right: Int): Tile = left.localMultiply(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/NormalizedDifference.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/NormalizedDifference.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala index 5760582d6..e62ccfc37 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/NormalizedDifference.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala @@ -19,10 +19,11 @@ * */ -package astraea.spark.rasterframes.expressions.localops -import astraea.spark.rasterframes.expressions.fpTile -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryRasterOp +package org.locationtech.rasterframes.expressions.localops + +import org.locationtech.rasterframes.expressions.fpTile +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -37,11 +38,11 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback * right - second tile argument""", examples = """ Examples: - > SELECT _FUNC_(nir, red); + > SELECT _FUNC_(nir, red) as ndvi; ...""" ) case class NormalizedDifference(left: Expression, right: Expression) extends BinaryRasterOp with CodegenFallback { - override val nodeName: String = "normalized_difference" + override val nodeName: String = "rf_normalized_difference" override protected def op(left: Tile, right: Tile): Tile = { val diff = fpTile(left.localSubtract(right)) val sum = fpTile(left.localAdd(right)) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Resample.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Resample.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala index fd2ae2f29..6e752dfbc 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Resample.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala @@ -19,14 +19,13 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.DynamicExtractors.tileExtractor -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor import geotrellis.raster.Tile import geotrellis.raster.resample.NearestNeighbor -import org.apache.spark.sql.rf._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -48,7 +47,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class Resample(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "resample" + override val nodeName: String = "rf_resample" override protected def op(left: Tile, right: Tile): Tile = left.resample(right.cols, right.rows, NearestNeighbor) override protected def op(left: Tile, right: Double): Tile = left.resample((left.cols * right).toInt, (left.rows * right).toInt, NearestNeighbor) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Round.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala similarity index 87% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Round.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala index 010666e17..92f1c2f89 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Round.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class Round(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { - override def nodeName: String = "round" + override def nodeName: String = "rf_round" override def na: Any = null override protected def op(child: Tile): Tile = child.localRound() } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Subtract.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala similarity index 90% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Subtract.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala index 203bb578d..c09a7ea47 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Subtract.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala @@ -19,9 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +package org.locationtech.rasterframes.expressions.localops + +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -42,7 +43,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Subtract(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_subtract" + override val nodeName: String = "rf_local_subtract" override protected def op(left: Tile, right: Tile): Tile = left.localSubtract(right) override protected def op(left: Tile, right: Double): Tile = left.localSubtract(right) override protected def op(left: Tile, right: Int): Tile = left.localSubtract(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Unequal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Unequal.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala index f3342b9c6..48a1e3963 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/localops/Unequal.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.localops +package org.locationtech.rasterframes.expressions.localops -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -41,7 +41,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ...""" ) case class Unequal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { - override val nodeName: String = "local_unequal" + override val nodeName: String = "rf_local_unequal" override protected def op(left: Tile, right: Tile): Tile = left.localUnequal(right) override protected def op(left: Tile, right: Double): Tile = left.localUnequal(right) override protected def op(left: Tile, right: Int): Tile = left.localUnequal(right) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala similarity index 80% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala index e4c0bcc00..8a8b70e00 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala @@ -15,25 +15,28 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes -import astraea.spark.rasterframes.expressions.accessors._ -import astraea.spark.rasterframes.expressions.aggstats._ -import astraea.spark.rasterframes.expressions.generators._ -import astraea.spark.rasterframes.expressions.localops._ -import astraea.spark.rasterframes.expressions.tilestats._ -import astraea.spark.rasterframes.expressions.transformers._ import geotrellis.raster.{DoubleConstantNoDataCellType, Tile} -import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection} import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF} +import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection} import org.apache.spark.sql.rf.VersionShims._ import org.apache.spark.sql.{SQLContext, rf} +import org.locationtech.rasterframes.expressions.accessors._ +import org.locationtech.rasterframes.expressions.aggregates.CellCountAggregate.DataCells +import org.locationtech.rasterframes.expressions.aggregates._ +import org.locationtech.rasterframes.expressions.generators._ +import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.tilestats._ +import org.locationtech.rasterframes.expressions.transformers._ -import scala.util.Try import scala.reflect.runtime.universe._ +import scala.util.Try /** * Module of Catalyst expressions for efficiently working with tiles. * @@ -61,11 +64,17 @@ package object expressions { registry.registerExpression[Add]("rf_local_add") registry.registerExpression[Subtract]("rf_local_subtract") + registry.registerExpression[TileAssembler]("rf_assemble_tile") registry.registerExpression[ExplodeTiles]("rf_explode_tiles") registry.registerExpression[GetCellType]("rf_cell_type") registry.registerExpression[SetCellType]("rf_convert_cell_type") - registry.registerExpression[GetDimensions]("rf_tile_dimensions") - registry.registerExpression[BoundsToGeometry]("rf_bounds_geometry") + registry.registerExpression[GetDimensions]("rf_dimensions") + registry.registerExpression[ExtentToGeometry]("st_geometry") + registry.registerExpression[GetGeometry]("rf_geometry") + registry.registerExpression[GeometryToExtent]("st_extent") + registry.registerExpression[GetExtent]("rf_extent") + registry.registerExpression[GetCRS]("rf_crs") + registry.registerExpression[RealizeTile]("rf_tile") registry.registerExpression[Subtract]("rf_local_subtract") registry.registerExpression[Multiply]("rf_local_multiply") registry.registerExpression[Divide]("rf_local_divide") @@ -78,6 +87,7 @@ package object expressions { registry.registerExpression[Unequal]("rf_local_unequal") registry.registerExpression[Sum]("rf_tile_sum") registry.registerExpression[Round]("rf_round") + registry.registerExpression[Abs]("rf_abs") registry.registerExpression[Log]("rf_log") registry.registerExpression[Log10]("rf_log10") registry.registerExpression[Log2]("rf_log2") @@ -92,12 +102,14 @@ package object expressions { registry.registerExpression[DataCells]("rf_data_cells") registry.registerExpression[NoDataCells]("rf_no_data_cells") registry.registerExpression[IsNoDataTile]("rf_is_no_data_tile") + registry.registerExpression[Exists]("rf_exists") + registry.registerExpression[ForAll]("rf_for_all") registry.registerExpression[TileMin]("rf_tile_min") registry.registerExpression[TileMax]("rf_tile_max") registry.registerExpression[TileMean]("rf_tile_mean") registry.registerExpression[TileStats]("rf_tile_stats") registry.registerExpression[TileHistogram]("rf_tile_histogram") - registry.registerExpression[CellCountAggregate.DataCells]("rf_agg_data_cells") + registry.registerExpression[DataCells]("rf_agg_data_cells") registry.registerExpression[CellCountAggregate.NoDataCells]("rf_agg_no_data_cells") registry.registerExpression[CellStatsAggregate.CellStatsAggregateUDAF]("rf_agg_stats") registry.registerExpression[HistogramAggregate.HistogramAggregateUDAF]("rf_agg_approx_histogram") @@ -110,9 +122,11 @@ package object expressions { registry.registerExpression[Mask.MaskByDefined]("rf_mask") registry.registerExpression[Mask.MaskByValue]("rf_mask_by_value") + registry.registerExpression[Mask.InverseMaskByValue]("rf_inverse_mask_by_value") registry.registerExpression[Mask.InverseMaskByDefined]("rf_inverse_mask") registry.registerExpression[DebugRender.RenderAscii]("rf_render_ascii") registry.registerExpression[DebugRender.RenderMatrix]("rf_render_matrix") + registry.registerExpression[transformers.ReprojectGeometry]("st_reproject") } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/DataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/DataCells.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala index a7d49c4ae..a18148db3 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/DataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala @@ -19,14 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{UnaryRasterOp, NullToValue} -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions.tilestats + +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.{DataType, LongType} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Counts the number of non-no-data cells in a tile", @@ -40,13 +41,13 @@ import org.apache.spark.sql.types.{DataType, LongType} ) case class DataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue { - override def nodeName: String = "data_cells" + override def nodeName: String = "rf_data_cells" override def dataType: DataType = LongType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = DataCells.op(tile) override def na: Any = 0L } object DataCells { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc def apply(tile: Column): TypedColumn[Any, Long] = new Column(DataCells(tile.expr)).as[Long] diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala new file mode 100644 index 000000000..cd04b1467 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala @@ -0,0 +1,47 @@ +package org.locationtech.rasterframes.expressions.tilestats + +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.isCellTrue +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext +import spire.syntax.cfor.cfor + +@ExpressionDescription( + usage = "_FUNC_(tile) - Returns true if any cells in the tile are true (non-zero and not nodata).", + arguments = + """ + Arguments: + * tile - tile to check + """, + examples = + """ + > SELECT _FUNC_(tile); + true + """ +) +case class Exists(child: Expression) extends UnaryRasterOp with CodegenFallback { + override def nodeName: String = "exists" + override def dataType: DataType = BooleanType + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile) + +} + +object Exists{ + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc + + def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(Exists(tile.expr)).as[Boolean] + + def op(tile: Tile): Boolean = { + cfor(0)(_ < tile.rows, _ + 1) { r ⇒ + cfor(0)(_ < tile.cols, _ + 1) { c ⇒ + if(tile.cellType.isFloatingPoint) { if(isCellTrue(tile.getDouble(c, r))) return true } + else { if(isCellTrue(tile.get(c, r))) return true } + } + } + false + } +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala new file mode 100644 index 000000000..a912a8a0b --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala @@ -0,0 +1,51 @@ +package org.locationtech.rasterframes.expressions.tilestats + +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.isCellTrue +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext +import spire.syntax.cfor.cfor + +@ExpressionDescription( + usage = "_FUNC_(tile) - Returns true if all cells in the tile are true (non-zero and not nodata).", + arguments = + """ + Arguments: + * tile - tile to check + """, + examples = + """ + > SELECT _FUNC_(tile); + true + """ +) +case class ForAll(child: Expression) extends UnaryRasterOp with CodegenFallback { + override def nodeName: String = "for_all" + override def dataType: DataType = BooleanType + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile) + +} + +object ForAll { + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc + + def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(ForAll(tile.expr)).as[Boolean] + + def op(tile: Tile): Boolean = { + cfor(0)(_ < tile.rows, _ + 1) { r ⇒ + cfor(0)(_ < tile.cols, _ + 1) { c ⇒ + if (tile.cellType.isFloatingPoint) { + if (!isCellTrue(tile.getDouble(c, r))) return false + } + else { + if (!isCellTrue(tile.get(c, r))) return false + } + } + } + true + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/IsNoDataTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/IsNoDataTile.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala index 7b360a07c..fd855cd39 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/IsNoDataTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala @@ -19,14 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{NullToValue, UnaryRasterOp} -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions.tilestats + +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{BooleanType, DataType} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Produces `true` if all the cells in a given tile are no-data", @@ -40,13 +41,13 @@ import org.apache.spark.sql.types.{BooleanType, DataType} ) case class IsNoDataTile(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue { - override def nodeName: String = "is_no_data_tile" + override def nodeName: String = "rf_is_no_data_tile" override def na: Any = true override def dataType: DataType = BooleanType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile.isNoDataTile } object IsNoDataTile { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(IsNoDataTile(tile.expr)).as[Boolean] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/NoDataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/NoDataCells.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala index 89c2ae10b..cf47ba14e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/NoDataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala @@ -19,15 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{UnaryRasterOp, NullToValue} -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.{DataType, LongType} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Counts the number of no-data cells in a tile", @@ -41,13 +41,13 @@ import org.apache.spark.sql.types.{DataType, LongType} ) case class NoDataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue { - override def nodeName: String = "no_data_cells" + override def nodeName: String = "rf_no_data_cells" override def dataType: DataType = LongType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = NoDataCells.op(tile) override def na: Any = 0L } object NoDataCells { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc def apply(tile: Column): TypedColumn[Any, Long] = new Column(NoDataCells(tile.expr)).as[Long] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/Sum.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/Sum.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala index cfa10666b..096acdab6 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/Sum.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala @@ -19,17 +19,18 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions.tilestats + +import org.locationtech.rasterframes.expressions.UnaryRasterOp import geotrellis.raster._ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.{DataType, DoubleType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( - usage = "_FUNC_(tile) - Computes the sum of all the cells in a tile..", + usage = "_FUNC_(tile) - Computes the sum of all the cells in a tile.", arguments = """ Arguments: * tile - tile to sum up""", @@ -39,13 +40,13 @@ import org.apache.spark.sql.{Column, TypedColumn} 2135.34""" ) case class Sum(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def nodeName: String = "tile_sum" + override def nodeName: String = "rf_tile_sum" override def dataType: DataType = DoubleType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile) } object Sum { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc def apply(tile: Column): TypedColumn[Any, Double] = new Column(Sum(tile.expr)).as[Double] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileHistogram.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala index d7fe7d0c1..96e3d3dcc 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileHistogram.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala @@ -19,17 +19,18 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.stats.CellHistogram +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.stats.CellHistogram import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Computes per-tile histogram.", @@ -43,7 +44,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class TileHistogram(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def nodeName: String = "tile_histogram" + override def nodeName: String = "rf_tile_histogram" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileHistogram.converter(TileHistogram.op(tile)) override def dataType: DataType = CellHistogram.schema diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMax.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala index 0e2595b2a..3204f4aaf 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala @@ -19,15 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{NullToValue, UnaryRasterOp} -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{DataType, DoubleType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Determines the maximum cell value.", @@ -41,13 +41,13 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class TileMax(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { - override def nodeName: String = "tile_max" + override def nodeName: String = "rf_tile_max" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile) override def dataType: DataType = DoubleType override def na: Any = Double.MinValue } object TileMax { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMax(tile.expr)).as[Double] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMean.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala index e23e68c08..92c833f98 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala @@ -19,16 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{NullToValue, UnaryRasterOp} -import astraea.spark.rasterframes.functions.safeEval -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{DataType, DoubleType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Computes the mean cell value of a tile.", @@ -42,13 +41,13 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class TileMean(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { - override def nodeName: String = "tile_mean" + override def nodeName: String = "rf_tile_mean" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile) override def dataType: DataType = DoubleType override def na: Any = Double.NaN } object TileMean { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMean(tile.expr)).as[Double] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala similarity index 85% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMin.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala index 4d2edc9b3..71fa0194a 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala @@ -19,15 +19,15 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.{NullToValue, UnaryRasterOp} -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{DataType, DoubleType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Determines the minimum cell value.", @@ -41,13 +41,13 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class TileMin(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { - override def nodeName: String = "tile_min" + override def nodeName: String = "rf_tile_min" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile) override def dataType: DataType = DoubleType override def na: Any = Double.MaxValue } object TileMin { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMin(tile.expr)).as[Double] diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileStats.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileStats.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala index 015f048e8..fac6d330e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/tilestats/TileStats.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala @@ -19,17 +19,18 @@ * */ -package astraea.spark.rasterframes.expressions.tilestats +package org.locationtech.rasterframes.expressions.tilestats -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.stats.CellStatistics +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.stats.CellStatistics import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Computes per-tile descriptive statistics.", @@ -43,7 +44,7 @@ import org.apache.spark.sql.{Column, TypedColumn} ) case class TileStats(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def nodeName: String = "tile_stats" + override def nodeName: String = "rf_tile_stats" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileStats.converter(TileStats.op(tile).orNull) override def dataType: DataType = CellStatistics.schema diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/DebugRender.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/DebugRender.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala index c26cc6b51..babb9c7b7 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/DebugRender.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala @@ -19,10 +19,10 @@ * */ -package astraea.spark.rasterframes.expressions.transformers -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.util.TileAsMatrix +package org.locationtech.rasterframes.expressions.transformers + +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.util.TileAsMatrix import geotrellis.raster.Tile import geotrellis.raster.render.ascii.AsciiArtEncoder import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types.{DataType, StringType} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String +import org.locationtech.rasterframes.model.TileContext abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp with CodegenFallback with Serializable { @@ -45,7 +46,7 @@ abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp } object DebugRender { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.stringEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.stringEnc @ExpressionDescription( usage = "_FUNC_(tile) - Coverts the contents of the given tile an ASCII art string rendering", @@ -54,7 +55,7 @@ object DebugRender { * tile - tile to render""" ) case class RenderAscii(child: Expression) extends DebugRender(true) { - override def nodeName: String = "render_ascii" + override def nodeName: String = "rf_render_ascii" } object RenderAscii { def apply(tile: Column): TypedColumn[Any, String] = @@ -68,7 +69,7 @@ object DebugRender { * tile - tile to render""" ) case class RenderMatrix(child: Expression) extends DebugRender(false) { - override def nodeName: String = "render_matrix" + override def nodeName: String = "rf_render_matrix" } object RenderMatrix { def apply(tile: Column): TypedColumn[Any, String] = diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/BoundsToGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala similarity index 76% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/BoundsToGeometry.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala index 9d6a8c652..9d2d12d2f 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/BoundsToGeometry.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala @@ -19,12 +19,11 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.row -import com.vividsolutions.jts.geom.{Envelope, Geometry} +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.row +import org.locationtech.jts.geom.{Envelope, Geometry} import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} @@ -40,13 +39,13 @@ import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders * * @since 8/24/18 */ -case class BoundsToGeometry(child: Expression) extends UnaryExpression with CodegenFallback { - override def nodeName: String = "bounds_geometry" +case class ExtentToGeometry(child: Expression) extends UnaryExpression with CodegenFallback { + override def nodeName: String = "st_geometry" override def dataType: DataType = JTSTypes.GeometryTypeInstance - private val envSchema = CatalystSerializer[Envelope].schema - private val extSchema = CatalystSerializer[Extent].schema + private val envSchema = schemaOf[Envelope] + private val extSchema = schemaOf[Extent] override def checkInputDataTypes(): TypeCheckResult = { child.dataType match { @@ -71,7 +70,7 @@ case class BoundsToGeometry(child: Expression) extends UnaryExpression with Code } } -object BoundsToGeometry extends SpatialEncoders { +object ExtentToGeometry extends SpatialEncoders { def apply(bounds: Column): TypedColumn[Any, Geometry] = - new Column(new BoundsToGeometry(bounds.expr)).as[Geometry] + new Column(new ExtentToGeometry(bounds.expr)).as[Geometry] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/GeometryToBounds.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala similarity index 76% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/GeometryToBounds.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala index 4e08ad9ea..adb52468b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/GeometryToBounds.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala @@ -19,10 +19,9 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} @@ -37,10 +36,10 @@ import org.apache.spark.sql.{Column, TypedColumn} * * @since 8/24/18 */ -case class GeometryToBounds(child: Expression) extends UnaryExpression with CodegenFallback { - override def nodeName: String = "geometry_bounds" +case class GeometryToExtent(child: Expression) extends UnaryExpression with CodegenFallback { + override def nodeName: String = "st_extent" - override def dataType: DataType = CatalystSerializer[Extent].schema + override def dataType: DataType = schemaOf[Extent] override def checkInputDataTypes(): TypeCheckResult = { child.dataType match { @@ -54,13 +53,13 @@ case class GeometryToBounds(child: Expression) extends UnaryExpression with Code override protected def nullSafeEval(input: Any): Any = { val geom = JTSTypes.GeometryTypeInstance.deserialize(input) val extent = Extent(geom.getEnvelopeInternal) - CatalystSerializer[Extent].toInternalRow(extent) + extent.toInternalRow } } -object GeometryToBounds { - import astraea.spark.rasterframes.encoders.StandardEncoders._ +object GeometryToExtent { + import org.locationtech.rasterframes.encoders.StandardEncoders._ def apply(bounds: Column): TypedColumn[Any, Extent] = - new Column(new GeometryToBounds(bounds.expr)).as[Extent] + new Column(new GeometryToExtent(bounds.expr)).as[Extent] } \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/Mask.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala similarity index 79% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/Mask.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala index 03e81efc2..106a52a7b 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/Mask.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala @@ -19,10 +19,11 @@ * */ -package astraea.spark.rasterframes.expressions.transformers -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.DynamicExtractors._ -import astraea.spark.rasterframes.expressions.row +package org.locationtech.rasterframes.expressions.transformers + +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.expressions.row import com.typesafe.scalalogging.LazyLogging import geotrellis.raster import geotrellis.raster.Tile @@ -81,7 +82,7 @@ abstract class Mask(val left: Expression, val middle: Expression, val right: Exp } } object Mask { - import astraea.spark.rasterframes.encoders.StandardEncoders.singlebandTileEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder @ExpressionDescription( usage = "_FUNC_(target, mask) - Generate a tile with the values from the data tile, but where cells in the masking tile contain NODATA, replace the data value with NODATA.", @@ -96,7 +97,7 @@ object Mask { ) case class MaskByDefined(target: Expression, mask: Expression) extends Mask(target, mask, Literal(0), false) { - override def nodeName: String = "mask" + override def nodeName: String = "rf_mask" } object MaskByDefined { def apply(targetTile: Column, maskTile: Column): TypedColumn[Any, Tile] = @@ -116,7 +117,7 @@ object Mask { ) case class InverseMaskByDefined(leftTile: Expression, rightTile: Expression) extends Mask(leftTile, rightTile, Literal(0), true) { - override def nodeName: String = "inverse_mask" + override def nodeName: String = "rf_inverse_mask" } object InverseMaskByDefined { def apply(srcTile: Column, maskingTile: Column): TypedColumn[Any, Tile] = @@ -136,10 +137,32 @@ object Mask { ) case class MaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression) extends Mask(leftTile, rightTile, maskValue, false) { - override def nodeName: String = "mask_by_value" + override def nodeName: String = "rf_mask_by_value" } object MaskByValue { def apply(srcTile: Column, maskingTile: Column, maskValue: Column): TypedColumn[Any, Tile] = new Column(MaskByValue(srcTile.expr, maskingTile.expr, maskValue.expr)).as[Tile] } + + @ExpressionDescription( + usage = "_FUNC_(target, mask, maskValue) - Generate a tile with the values from the data tile, but where cells in the masking tile DO NOT contain the masking value, replace the data value with NODATA.", + arguments = """ + Arguments: + * target - tile to mask + * mask - masking definition + * maskValue - value in the `mask` for which to mark `target` as data cells + """, + examples = """ + Examples: + > SELECT _FUNC_(target, mask, maskValue); + ...""" + ) + case class InverseMaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression) + extends Mask(leftTile, rightTile, maskValue, true) { + override def nodeName: String = "rf_inverse_mask_by_value" + } + object InverseMaskByValue { + def apply(srcTile: Column, maskingTile: Column, maskValue: Column): TypedColumn[Any, Tile] = + new Column(InverseMaskByValue(srcTile.expr, maskingTile.expr, maskValue.expr)).as[Tile] + } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterRefToTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala similarity index 70% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterRefToTile.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala index c3aa3f337..f0c82c6de 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/RasterRefToTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala @@ -19,19 +19,18 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.expressions.row -import astraea.spark.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.row import com.typesafe.scalalogging.LazyLogging -import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression} import org.apache.spark.sql.rf._ import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.tiles.ProjectedRasterTile /** * Realizes a RasterRef into a Tile. @@ -43,19 +42,18 @@ case class RasterRefToTile(child: Expression) extends UnaryExpression override def nodeName: String = "raster_ref_to_tile" - override def inputTypes = Seq(CatalystSerializer[RasterRef].schema) + override def inputTypes = Seq(schemaOf[RasterRef]) - override def dataType: DataType = new TileUDT + override def dataType: DataType = schemaOf[ProjectedRasterTile] override protected def nullSafeEval(input: Any): Any = { implicit val ser = TileUDT.tileSerializer val ref = row(input).to[RasterRef] - (ref.tile: Tile).toInternalRow + ref.tile.toInternalRow } } object RasterRefToTile { - import astraea.spark.rasterframes.encoders.StandardEncoders._ - def apply(rr: Column): TypedColumn[Any, Tile] = - new Column(RasterRefToTile(rr.expr)).as[Tile] + def apply(rr: Column): TypedColumn[Any, ProjectedRasterTile] = + new Column(RasterRefToTile(rr.expr)).as[ProjectedRasterTile] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/ReprojectGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala similarity index 53% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/ReprojectGeometry.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala index 7e78c5942..9c1ab2234 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/ReprojectGeometry.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala @@ -19,37 +19,54 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.encoders.{CatalystSerializer, serialized_literal} -import astraea.spark.rasterframes.jts.ReprojectionTransformer -import com.vividsolutions.jts.geom.Geometry +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.serialized_literal +import org.locationtech.jts.geom.Geometry import geotrellis.proj4.CRS import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.jts.JTSTypes +import org.apache.spark.sql.jts.{AbstractGeometryUDT, JTSTypes} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.expressions.DynamicExtractors +import org.locationtech.rasterframes.jts.ReprojectionTransformer +import org.locationtech.rasterframes.model.LazyCRS -/** - * - * - * @since 11/29/18 - */ +@ExpressionDescription( + usage = "_FUNC_(geom, srcCRS, dstCRS) - Reprojects the given `geom` from `srcCRS` to `dstCRS", + arguments = """ + Arguments: + * geom - the geometry column to reproject + * srcCRS - the CRS of the `geom` column + * dstCRS - the CRS to project geometry into""", + examples = """ + Examples: + > SELECT _FUNC_(geom, srcCRS, dstCRS); + ...""" +) case class ReprojectGeometry(geometry: Expression, srcCRS: Expression, dstCRS: Expression) extends Expression - with CodegenFallback with ExpectsInputTypes { + with CodegenFallback { - override def nodeName: String = "reproject_geometry" + override def nodeName: String = "st_reproject" override def dataType: DataType = JTSTypes.GeometryTypeInstance override def nullable: Boolean = geometry.nullable || srcCRS.nullable || dstCRS.nullable override def children: Seq[Expression] = Seq(geometry, srcCRS, dstCRS) - private def crsSerde = CatalystSerializer[CRS] - override val inputTypes = Seq( - dataType, crsSerde.schema, crsSerde.schema - ) + + override def checkInputDataTypes(): TypeCheckResult = { + if (!geometry.dataType.isInstanceOf[AbstractGeometryUDT[_]]) + TypeCheckFailure(s"Input type '${geometry.dataType}' does not conform to a geometry type.") + else if(!DynamicExtractors.crsExtractor.isDefinedAt(srcCRS.dataType)) + TypeCheckFailure(s"Input type '${srcCRS.dataType}' cannot be interpreted as a CRS.") + else if(!DynamicExtractors.crsExtractor.isDefinedAt(dstCRS.dataType)) + TypeCheckFailure(s"Input type '${dstCRS.dataType}' cannot be interpreted as a CRS.") + else TypeCheckResult.TypeCheckSuccess + } /** Reprojects a geometry column from one CRS to another. */ val reproject: (Geometry, CRS, CRS) ⇒ Geometry = @@ -59,10 +76,15 @@ case class ReprojectGeometry(geometry: Expression, srcCRS: Expression, dstCRS: E } override def eval(input: InternalRow): Any = { - val geom = JTSTypes.GeometryTypeInstance.deserialize(geometry.eval(input)) - val src = srcCRS.eval(input).asInstanceOf[InternalRow].to[CRS] - val dst = dstCRS.eval(input).asInstanceOf[InternalRow].to[CRS] - JTSTypes.GeometryTypeInstance.serialize(reproject(geom, src, dst)) + val src = DynamicExtractors.crsExtractor(srcCRS.dataType)(srcCRS.eval(input)) + val dst = DynamicExtractors.crsExtractor(dstCRS.dataType)(dstCRS.eval(input)) + (src, dst) match { + // Optimized pass-through case. + case (s: LazyCRS, r: LazyCRS) if s.encoded == r.encoded => geometry.eval(input) + case _ => + val geom = JTSTypes.GeometryTypeInstance.deserialize(geometry.eval(input)) + JTSTypes.GeometryTypeInstance.serialize(reproject(geom, src, dst)) + } } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/SetCellType.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/SetCellType.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala index 96fcd4288..6990e34df 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/SetCellType.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala @@ -19,13 +19,12 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.encoders.StandardEncoders._ -import astraea.spark.rasterframes.expressions.DynamicExtractors.tileExtractor -import astraea.spark.rasterframes.expressions.row +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.StandardEncoders._ +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.expressions.row import geotrellis.raster.{CellType, Tile} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult @@ -33,7 +32,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression} import org.apache.spark.sql.functions.lit -import org.apache.spark.sql.rf._ +import org.apache.spark.sql.rf.{TileUDT, WithTypeConformity} import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String @@ -50,7 +49,7 @@ case class SetCellType(tile: Expression, cellType: Expression) override def nodeName: String = "set_cell_type" override def dataType: DataType = left.dataType - private val ctSchema = CatalystSerializer[CellType].schema + private val ctSchema = schemaOf[CellType] override def checkInputDataTypes(): TypeCheckResult = { if (!tileExtractor.isDefinedAt(left.dataType)) diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayDouble.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayDouble.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala index 02a4bc4e8..5d7786f1c 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayDouble.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala @@ -19,15 +19,16 @@ * */ -package astraea.spark.rasterframes.expressions.transformers -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext +package org.locationtech.rasterframes.expressions.transformers + +import org.locationtech.rasterframes.expressions.UnaryRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{DataType, DataTypes, DoubleType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Coverts the contents of the given tile to an array of double floating-point values", @@ -36,14 +37,14 @@ import org.apache.spark.sql.{Column, TypedColumn} * tile - tile to convert""" ) case class TileToArrayDouble(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def nodeName: String = "tile_to_array_double" + override def nodeName: String = "rf_tile_to_array_double" override def dataType: DataType = DataTypes.createArrayType(DoubleType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { ArrayData.toArrayData(tile.toArrayDouble()) } } object TileToArrayDouble { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc def apply(tile: Column): TypedColumn[Any, Array[Double]] = new Column(TileToArrayDouble(tile.expr)).as[Array[Double]] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayInt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala similarity index 81% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayInt.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala index 31ad81516..c299d57c7 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/TileToArrayInt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala @@ -19,16 +19,17 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers -import astraea.spark.rasterframes.expressions.UnaryRasterOp -import astraea.spark.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.UnaryRasterOp import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType} import org.apache.spark.sql.{Column, TypedColumn} +import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( usage = "_FUNC_(tile) - Coverts the contents of the given tile to an array of integer values", @@ -37,14 +38,14 @@ import org.apache.spark.sql.{Column, TypedColumn} * tile - tile to convert""" ) case class TileToArrayInt(child: Expression) extends UnaryRasterOp with CodegenFallback { - override def nodeName: String = "tile_to_array_int" + override def nodeName: String = "rf_tile_to_array_int" override def dataType: DataType = DataTypes.createArrayType(IntegerType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { ArrayData.toArrayData(tile.toArray()) } } object TileToArrayInt { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc + import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc def apply(tile: Column): TypedColumn[Any, Array[Int]] = new Column(TileToArrayInt(tile.expr)).as[Array[Int]] } diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/URIToRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala similarity index 61% rename from core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/URIToRasterSource.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala index 0821e43db..903e62dde 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/expressions/transformers/URIToRasterSource.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala @@ -19,20 +19,18 @@ * */ -package astraea.spark.rasterframes.expressions.transformers +package org.locationtech.rasterframes.expressions.transformers import java.net.URI -import astraea.spark.rasterframes.ref.RasterSource.ReadCallback -import astraea.spark.rasterframes.ref.{RasterRef, RasterSource} +import org.locationtech.rasterframes.RasterSourceType import com.typesafe.scalalogging.LazyLogging import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression} -import org.apache.spark.sql.rf._ import org.apache.spark.sql.types.{DataType, StringType} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String - +import org.locationtech.rasterframes.ref.RasterSource /** * Catalyst generator to convert a geotiff download URL into a series of rows @@ -40,28 +38,24 @@ import org.apache.spark.unsafe.types.UTF8String * * @since 5/4/18 */ -case class URIToRasterSource(override val child: Expression, accumulator: Option[ReadCallback]) +case class URIToRasterSource(override val child: Expression) extends UnaryExpression with ExpectsInputTypes with CodegenFallback with LazyLogging { - override def nodeName: String = "uri_to_raster_source" + override def nodeName: String = "rf_uri_to_raster_source" - override def dataType: DataType = new RasterSourceUDT + override def dataType: DataType = RasterSourceType override def inputTypes = Seq(StringType) override protected def nullSafeEval(input: Any): Any = { val uriString = input.asInstanceOf[UTF8String].toString val uri = URI.create(uriString) - val ref = RasterSource(uri, accumulator) - RasterSourceUDT.serialize(ref) + val ref = RasterSource(uri) + RasterSourceType.serialize(ref) } } object URIToRasterSource { - def apply(rasterURI: Column): TypedColumn[Any, RasterRef] = - new Column(new URIToRasterSource(rasterURI.expr, None)).as[RasterRef] - def apply(rasterURI: Column, accumulator: ReadCallback): TypedColumn[Any, RasterRef] = - new Column(new URIToRasterSource(rasterURI.expr, Option(accumulator))).as[RasterRef] - def apply(rasterURI: Column, accumulator: Option[ReadCallback]): TypedColumn[Any, RasterRef] = - new Column(new URIToRasterSource(rasterURI.expr, accumulator)).as[RasterRef] + def apply(rasterURI: Column): TypedColumn[Any, RasterSource] = + new Column(new URIToRasterSource(rasterURI.expr)).as[RasterSource] } diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala new file mode 100644 index 000000000..7bf3230b3 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala @@ -0,0 +1,73 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions + +import org.locationtech.rasterframes.PairRDDConverter._ +import org.locationtech.rasterframes.StandardColumns._ +import Implicits._ +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.RasterFrameLayer +import geotrellis.raster.CellGrid +import geotrellis.spark._ +import geotrellis.spark.io._ +import geotrellis.util.MethodExtensions +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SparkSession +import org.locationtech.rasterframes.PairRDDConverter + +/** + * Extension method on `ContextRDD`-shaped RDDs with appropriate context bounds to create a RasterFrameLayer. + * @since 7/18/17 + */ +abstract class SpatialContextRDDMethods[T <: CellGrid](implicit spark: SparkSession) + extends MethodExtensions[RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]] { + import PairRDDConverter._ + + def toLayer(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = toLayer(TILE_COLUMN.columnName) + + def toLayer(tileColumnName: String)(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = { + val df = self.toDataFrame.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata) + val defName = TILE_COLUMN.columnName + df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName)) + .certify + } +} + +/** + * Extension method on `ContextRDD`-shaped `Tile` RDDs keyed with [[SpaceTimeKey]], with appropriate context bounds to create a RasterFrameLayer. + * @since 9/11/17 + */ +abstract class SpatioTemporalContextRDDMethods[T <: CellGrid]( + implicit spark: SparkSession) + extends MethodExtensions[RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] { + + def toLayer(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrameLayer = toLayer(TILE_COLUMN.columnName) + + def toLayer(tileColumnName: String)(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrameLayer = { + val df = self.toDataFrame + .setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata) + .setTemporalColumnRole(TEMPORAL_KEY_COLUMN) + val defName = TILE_COLUMN.columnName + df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName)) + .certify + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala new file mode 100644 index 000000000..1e94ff3ca --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala @@ -0,0 +1,305 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions + +import geotrellis.proj4.CRS +import geotrellis.spark.io._ +import geotrellis.spark.{SpaceTimeKey, SpatialComponent, SpatialKey, TemporalKey, TileLayerMetadata} +import geotrellis.util.MethodExtensions +import geotrellis.vector.Extent +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.types.{MetadataBuilder, StructField} +import org.apache.spark.sql.{Column, DataFrame, TypedColumn} +import org.locationtech.rasterframes.StandardColumns._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.StandardEncoders._ +import org.locationtech.rasterframes.expressions.DynamicExtractors +import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.{MetadataKeys, RasterFrameLayer} +import spray.json.JsonFormat + +import scala.util.Try + +/** + * Extension methods over [[DataFrame]]. + * + * @since 7/18/17 + */ +trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with MetadataKeys { + import Implicits.{WithDataFrameMethods, WithMetadataBuilderMethods, WithMetadataMethods, WithRasterFrameLayerMethods} + + private def selector(column: Column) = (attr: Attribute) ⇒ + attr.name == column.columnName || attr.semanticEquals(column.expr) + + /** Map over the Attribute representation of Columns, modifying the one matching `column` with `op`. */ + private[rasterframes] def mapColumnAttribute(column: Column, op: Attribute ⇒ Attribute): DF = { + val analyzed = self.queryExecution.analyzed.output + val selects = selector(column) + val attrs = analyzed.map { attr ⇒ + if(selects(attr)) op(attr) else attr + } + self.select(attrs.map(a ⇒ new Column(a)): _*).asInstanceOf[DF] + } + + private[rasterframes] def addColumnMetadata(column: Column, op: MetadataBuilder ⇒ MetadataBuilder): DF = { + mapColumnAttribute(column, attr ⇒ { + val md = new MetadataBuilder().withMetadata(attr.metadata) + attr.withMetadata(op(md).build) + }) + } + + private[rasterframes] def fetchMetadataValue[D](column: Column, reader: (Attribute) ⇒ D): Option[D] = { + val analyzed = self.queryExecution.analyzed.output + analyzed.find(selector(column)).map(reader) + } + + private[rasterframes] + def setSpatialColumnRole[K: SpatialComponent: JsonFormat]( + column: Column, md: TileLayerMetadata[K]): DF = + addColumnMetadata(column, + _.attachContext(md.asColumnMetadata).tagSpatialKey + ) + + private[rasterframes] + def setTemporalColumnRole(column: Column): DF = + addColumnMetadata(column, _.tagTemporalKey) + + /** Get the role tag the column plays in the RasterFrameLayer, if any. */ + private[rasterframes] + def getColumnRole(column: Column): Option[String] = + fetchMetadataValue(column, _.metadata.getString(SPATIAL_ROLE_KEY)) + + /** Get the columns that are of type `Tile` */ + def tileColumns: Seq[Column] = + self.schema.fields + .filter(f => DynamicExtractors.tileExtractor.isDefinedAt(f.dataType)) + .map(f ⇒ self.col(f.name)) + + /** Get the columns that look like `ProjectedRasterTile`s. */ + def projRasterColumns: Seq[Column] = + self.schema.fields + .filter(_.dataType.conformsTo[ProjectedRasterTile]) + .map(f => self.col(f.name)) + + /** Get the columns that look like `Extent`s. */ + def extentColumns: Seq[Column] = + self.schema.fields + .filter(_.dataType.conformsTo[Extent]) + .map(f => self.col(f.name)) + + /** Get the columns that look like `CRS`s. */ + def crsColumns: Seq[Column] = + self.schema.fields + .filter(_.dataType.conformsTo[CRS]) + .map(f => self.col(f.name)) + + /** Get the columns that are not of type `Tile` */ + def notTileColumns: Seq[Column] = + self.schema.fields + .filter(f => !DynamicExtractors.tileExtractor.isDefinedAt(f.dataType)) + .map(f ⇒ self.col(f.name)) + + /** Get the spatial column. */ + def spatialKeyColumn: Option[TypedColumn[Any, SpatialKey]] = { + val key = findSpatialKeyField + key + .map(_.name) + .map(self.col(_).as[SpatialKey]) + } + + /** Get the temporal column, if any. */ + def temporalKeyColumn: Option[TypedColumn[Any, TemporalKey]] = { + val key = findTemporalKeyField + key.map(_.name).map(self.col(_).as[TemporalKey]) + } + + /** Find the field tagged with the requested `role` */ + private[rasterframes] def findRoleField(role: String): Option[StructField] = + self.schema.fields.find( + f ⇒ + f.metadata.contains(SPATIAL_ROLE_KEY) && + f.metadata.getString(SPATIAL_ROLE_KEY) == role + ) + + /** The spatial key is the first one found with context metadata attached to it. */ + private[rasterframes] def findSpatialKeyField: Option[StructField] = + findRoleField(SPATIAL_KEY_COLUMN.columnName) + + /** The temporal key is the first one found with the temporal tag. */ + private[rasterframes] def findTemporalKeyField: Option[StructField] = + findRoleField(TEMPORAL_KEY_COLUMN.columnName) + + /** Renames all columns such that they start with the given prefix string. + * Useful for preparing dataframes for joins where duplicate names may arise. + */ + def withPrefixedColumnNames(prefix: String): DF = + self.columns.foldLeft(self)((df, c) ⇒ df.withColumnRenamed(c, s"$prefix$c").asInstanceOf[DF]) + + /** + * Performs a jeft join on the dataframe `right` to this one, reprojecting and merging tiles as necessary. + * The operation is logically a "left outer" join, with the left side also determining the target CRS and extents. + * Right side may have multiple Tile columns. Assumes both dataframes use the column names `extent` and `crs` for + * the Extent and CRS details for each row. The join expression used is: + * + * {{{ + * st_intersects(st_geometry(leftExtent), st_reproject(st_geometry(rightExtent), rightCRS, leftCRS)) + * }}} + * + * @param right Right side of the join. + * @return joined dataframe + */ + def rasterJoin(right: DataFrame): DataFrame = RasterJoin(self, right) + + /** + * Performs a jeft join on the dataframe `right` to this one, reprojecting and merging tiles as necessary. + * The operation is logically a "left outer" join, with the left side also determining the target CRS and extents. + * Right side may have multiple Tile columns. This variant allows for the specific geospatial columns to be + * specified. The join expression used is: + * {{{ + * st_intersects(st_geometry(leftExtent), st_reproject(st_geometry(rightExtent), rightCRS, leftCRS)) + * }}} + * + * @param right right dataframe + * @param leftExtent this (left) dataframe's Extent column + * @param leftCRS this (left) datafrasme's CRS column + * @param rightExtent right dataframe's CRS extent + * @param rightCRS right dataframe's CRS column + * @return joined dataframe + */ + def rasterJoin(right: DataFrame, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = + RasterJoin(self, right, leftExtent, leftCRS, rightExtent, rightCRS) + + /** + * Performs a jeft join on the dataframe `right` to this one, reprojecting and merging tiles as necessary. + * The operation is logically a "left outer" join, with the left side also determining the target CRS and extents. + * Right side may have multiple Tile columns. This variant allows for the specific geospatial columns and join + * expression to be specified. + * + * @param right right dataframe + * @param leftExtent this (left) dataframe's Extent column + * @param joinExpr join expression + * @param leftCRS this (left) datafrasme's CRS column + * @param rightExtent right dataframe's CRS extent + * @param rightCRS right dataframe's CRS column + * @return joined dataframe + */ + def rasterJoin(right: DataFrame, joinExpr: Column, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = + RasterJoin(self, right, joinExpr, leftExtent, leftCRS, rightExtent, rightCRS) + + + /** Layout contents of RasterFrame to a layer. Assumes CRS and extent columns exist. */ + def toLayer(tlm: TileLayerMetadata[SpatialKey]): RasterFrameLayer = ReprojectToLayer(self, tlm) + + /** Coerces this DataFrame to a RasterFrameLayer after ensuring it has: + * + *
                + *
              1. a space or space-time key column + *
              2. one or more tile columns + *
              3. tile layout metadata + *
                  + * + * If any of the above are violated, and [[IllegalArgumentException]] is thrown. + * + * @return validated RasterFrameLayer + * @throws IllegalArgumentException when constraints are not met. + */ + @throws[IllegalArgumentException] + def asLayer: RasterFrameLayer = { + val potentialRF = certifyRasterframe(self) + + require( + potentialRF.findSpatialKeyField.nonEmpty, + "A RasterFrameLayer requires a column identified as a spatial key" + ) + + require(potentialRF.tileColumns.nonEmpty, "A RasterFrameLayer requires at least one tile column") + + require( + Try(potentialRF.tileLayerMetadata).isSuccess, + "A RasterFrameLayer requires embedded TileLayerMetadata" + ) + + potentialRF + } + + /** + * Convert DataFrame already in a uniform gridding into a RasterFrameLayer + * + * @param spatialKey The column where the spatial key is stored + * @param tlm Metadata describing layout under which tiles were created. Note: no checking is + * performed to ensure metadata, key-space, and tiles are coherent. + * @throws IllegalArgumentException when constraints outlined in `asLayer` are not met. + * @return Encoded RasterFrameLayer + */ + @throws[IllegalArgumentException] + private[rasterframes] + def asLayer(spatialKey: Column, tlm: TileLayerMetadata[SpatialKey]): RasterFrameLayer = + setSpatialColumnRole(spatialKey, tlm).asLayer + + /** + * Convert DataFrame already in a uniform gridding into a RasterFrameLayer + * + * @param spatialKey The column where the spatial key is stored + * @param temporalKey The column tagged under the temporal role + * @param tlm Metadata describing layout under which tiles were created. Note: no checking is + * performed to ensure metadata, key-space, and tiles are coherent. + * @throws IllegalArgumentException when constraints outlined in `asLayer` are not met. + * @return Encoded RasterFrameLayer + */ + @throws[IllegalArgumentException] + private[rasterframes] + def asLayer(spatialKey: Column, temporalKey: Column, tlm: TileLayerMetadata[SpaceTimeKey]): RasterFrameLayer = + setSpatialColumnRole(spatialKey, tlm) + .setTemporalColumnRole(temporalKey) + .asLayer + + /** + * Converts [[DataFrame]] to a RasterFrameLayer if the following constraints are fulfilled: + * + *
                    + *
                  1. a space or space-time key column + *
                  2. one or more tile columns + *
                  3. tile layout metadata + *
                      + * + * @return Some[RasterFrameLayer] if constraints fulfilled, [[None]] otherwise. + */ + def asLayerSafely: Option[RasterFrameLayer] = Try(asLayer).toOption + + /** + * Tests for the following conditions on the [[DataFrame]]: + * + *
                        + *
                      1. a space or space-time key column + *
                      2. one or more tile columns + *
                      3. tile layout metadata + *
                          + * + * @return true if all constraints are fulfilled, false otherwise. + */ + def isAlreadyLayer: Boolean = Try(asLayer).isSuccess + + /** Internal method for slapping the RasterFreameLayer seal of approval on a DataFrame. + * Only call if if you are sure it has a spatial key and tile columns and TileLayerMetadata. */ + private[rasterframes] def certify = certifyRasterframe(self) +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala similarity index 82% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala index 8fdda51a0..563e03e87 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala @@ -15,16 +15,18 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions -import astraea.spark.rasterframes.RasterFrame -import astraea.spark.rasterframes.util.{WithMergeMethods, WithPrototypeMethods} +import org.locationtech.rasterframes.RasterFrameLayer +import org.locationtech.rasterframes.util.{WithMergeMethods, WithPrototypeMethods} import geotrellis.raster._ +import geotrellis.raster.io.geotiff.SinglebandGeoTiff import geotrellis.spark.{Metadata, SpaceTimeKey, SpatialKey, TileLayerMetadata} import geotrellis.util.MethodExtensions -import org.apache.hadoop.conf.{Configuration => HadoopConfiguration} import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.sql._ @@ -50,9 +52,11 @@ trait Implicits { implicit class WithProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithPrototypeMethods: TypeTag]( val self: ProjectedRaster[T]) extends ProjectedRasterMethods[T] + implicit class WithSinglebandGeoTiffMethods(val self: SinglebandGeoTiff) extends SinglebandGeoTiffMethods + implicit class WithDataFrameMethods[D <: DataFrame](val self: D) extends DataFrameMethods[D] - implicit class WithRasterFrameMethods(val self: RasterFrame) extends RasterFrameMethods + implicit class WithRasterFrameLayerMethods(val self: RasterFrameLayer) extends RasterFrameLayerMethods implicit class WithSpatialContextRDDMethods[T <: CellGrid]( val self: RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]] @@ -62,17 +66,17 @@ trait Implicits { val self: RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]] )(implicit spark: SparkSession) extends SpatioTemporalContextRDDMethods[T] - private[astraea] + private[rasterframes] implicit class WithMetadataMethods[R: JsonFormat](val self: R) extends MetadataMethods[R] - private[astraea] + private[rasterframes] implicit class WithMetadataAppendMethods(val self: SMetadata) extends MethodExtensions[SMetadata] { def append = new MetadataBuilder().withMetadata(self) } - private[astraea] + private[rasterframes] implicit class WithMetadataBuilderMethods(val self: MetadataBuilder) extends MetadataBuilderMethods } diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/KryoMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/KryoMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala index 52ed69557..7b291d7d6 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/KryoMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala @@ -19,12 +19,12 @@ * */ -package astraea.spark.rasterframes.extensions -import astraea.spark.rasterframes.util.RFKryoRegistrator +package org.locationtech.rasterframes.extensions import geotrellis.util.MethodExtensions import org.apache.spark.SparkConf import org.apache.spark.serializer.KryoSerializer import org.apache.spark.sql.SparkSession +import org.locationtech.rasterframes.util.RFKryoRegistrator object KryoMethods { val kryoProperties = Map("spark.serializer" -> classOf[KryoSerializer].getName, diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataBuilderMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala similarity index 85% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataBuilderMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala index 491c30b4d..fc2401bb5 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataBuilderMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala @@ -15,14 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions -import astraea.spark.rasterframes.{MetadataKeys, StandardColumns} import geotrellis.util.MethodExtensions import org.apache.spark.sql.types.{Metadata, MetadataBuilder} -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.{MetadataKeys, StandardColumns} /** * Convenience to deal with boilerplate associated with adding @@ -30,7 +32,7 @@ import astraea.spark.rasterframes.util._ * * @since 12/21/17 */ -private[astraea] +private[rasterframes] abstract class MetadataBuilderMethods extends MethodExtensions[MetadataBuilder] with MetadataKeys with StandardColumns { def attachContext(md: Metadata) = self.putMetadata(CONTEXT_METADATA_KEY, md) def tagSpatialKey = self.putString(SPATIAL_ROLE_KEY, SPATIAL_KEY_COLUMN.columnName) diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala similarity index 60% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala index e1a886e60..5d96abdf4 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/MetadataMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala @@ -1,20 +1,26 @@ /* + * This software is licensed under the Apache 2 license, quoted below. + * * Copyright 2017 Astraea, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * [http://www.apache.org/licenses/LICENSE-2.0] * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions + import geotrellis.util.MethodExtensions import spray.json.{JsObject, JsonFormat} import org.apache.spark.sql.types.{Metadata ⇒ SQLMetadata} diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ProjectedRasterMethods.scala similarity index 66% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/ProjectedRasterMethods.scala index 96709ef10..81f5054f9 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ProjectedRasterMethods.scala @@ -1,20 +1,41 @@ -package astraea.spark.rasterframes.extensions +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions import java.time.ZonedDateTime -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.{PairRDDConverter, RasterFrame, StandardColumns} import geotrellis.raster.{CellGrid, ProjectedRaster} import geotrellis.spark._ import geotrellis.spark.tiling._ import geotrellis.util.MethodExtensions import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.{PairRDDConverter, RasterFrameLayer, StandardColumns} import scala.reflect.runtime.universe._ /** - * Extension methods on [[ProjectedRaster]] for creating [[RasterFrame]]s. + * Extension methods on [[ProjectedRaster]] for creating [[RasterFrameLayer]]s. * * @since 8/10/17 */ @@ -24,63 +45,64 @@ abstract class ProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithProto type XTileLayerRDD[K] = RDD[(K, T)] with Metadata[TileLayerMetadata[K]] /** - * Convert the wrapped [[ProjectedRaster]] into a [[RasterFrame]] with a + * Convert the wrapped [[ProjectedRaster]] into a [[RasterFrameLayer]] with a * single row. * - * @param spark [[SparkSession]] in which to create [[RasterFrame]] + * @param spark [[SparkSession]] in which to create [[RasterFrameLayer]] */ - def toRF(implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName) + def toLayer(implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = + toLayer(TILE_COLUMN.columnName) /** - * Convert the wrapped [[ProjectedRaster]] into a [[RasterFrame]] with a + * Convert the wrapped [[ProjectedRaster]] into a [[RasterFrameLayer]] with a * single row. * - * @param spark [[SparkSession]] in which to create [[RasterFrame]] + * @param spark [[SparkSession]] in which to create [[RasterFrameLayer]] */ - def toRF(tileColName: String) - (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = { + def toLayer(tileColName: String) + (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = { val (cols, rows) = self.raster.dimensions - toRF(cols, rows, tileColName) + toLayer(cols, rows, tileColName) } /** - * Convert the [[ProjectedRaster]] into a [[RasterFrame]] using the + * Convert the [[ProjectedRaster]] into a [[RasterFrameLayer]] using the * given dimensions as the target per-row tile size. * * @param tileCols Max number of horizontal cells per tile * @param tileRows Max number of vertical cells per tile - * @param spark [[SparkSession]] in which to create [[RasterFrame]] + * @param spark [[SparkSession]] in which to create [[RasterFrameLayer]] */ - def toRF(tileCols: Int, tileRows: Int) - (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = - toRF(tileCols, tileRows, TILE_COLUMN.columnName) + def toLayer(tileCols: Int, tileRows: Int) + (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = + toLayer(tileCols, tileRows, TILE_COLUMN.columnName) /** - * Convert the [[ProjectedRaster]] into a [[RasterFrame]] using the + * Convert the [[ProjectedRaster]] into a [[RasterFrameLayer]] using the * given dimensions as the target per-row tile size. * * @param tileCols Max number of horizontal cells per tile * @param tileRows Max number of vertical cells per tile * @param tileColName Name to give the created tile column - * @param spark [[SparkSession]] in which to create [[RasterFrame]] + * @param spark [[SparkSession]] in which to create [[RasterFrameLayer]] */ - def toRF(tileCols: Int, tileRows: Int, tileColName: String) - (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = { - toTileLayerRDD(tileCols, tileRows).toRF(tileColName) + def toLayer(tileCols: Int, tileRows: Int, tileColName: String) + (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = { + toTileLayerRDD(tileCols, tileRows).toLayer(tileColName) } /** - * Convert the [[ProjectedRaster]] into a [[RasterFrame]] using the + * Convert the [[ProjectedRaster]] into a [[RasterFrameLayer]] using the * given dimensions as the target per-row tile size and singular timestamp as the temporal component. * * @param tileCols Max number of horizontal cells per tile * @param tileRows Max number of vertical cells per tile. * @param timestamp Temporal key value to assign to tiles. - * @param spark [[SparkSession]] in which to create [[RasterFrame]] + * @param spark [[SparkSession]] in which to create [[RasterFrameLayer]] */ - def toRF(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime) - (implicit spark: SparkSession, schema: PairRDDConverter[SpaceTimeKey, T]): RasterFrame = - toTileLayerRDD(tileCols, tileRows, timestamp).toRF + def toLayer(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime) + (implicit spark: SparkSession, schema: PairRDDConverter[SpaceTimeKey, T]): RasterFrameLayer = + toTileLayerRDD(tileCols, tileRows, timestamp).toLayer /** * Convert the [[ProjectedRaster]] into a [[TileLayerRDD[SpatialKey]] using the @@ -92,7 +114,7 @@ abstract class ProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithProto */ def toTileLayerRDD(tileCols: Int, tileRows: Int)(implicit spark: SparkSession): XTileLayerRDD[SpatialKey] = { - val layout = LayoutDefinition(self.rasterExtent, tileCols, tileRows) + val layout = LayoutDefinition(self.raster.rasterExtent, tileCols, tileRows) val kb = KeyBounds(SpatialKey(0, 0), SpatialKey(layout.layoutCols - 1, layout.layoutRows - 1)) val tlm = TileLayerMetadata(self.tile.cellType, layout, self.extent, self.crs, kb) @@ -115,7 +137,7 @@ abstract class ProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithProto * @param spark [[SparkSession]] in which to create RDD */ def toTileLayerRDD(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime)(implicit spark: SparkSession): XTileLayerRDD[SpaceTimeKey] = { - val layout = LayoutDefinition(self.rasterExtent, tileCols, tileRows) + val layout = LayoutDefinition(self.raster.rasterExtent, tileCols, tileRows) val kb = KeyBounds(SpaceTimeKey(0, 0, timestamp), SpaceTimeKey(layout.layoutCols - 1, layout.layoutRows - 1, timestamp)) val tlm = TileLayerMetadata(self.tile.cellType, layout, self.extent, self.crs, kb) diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/RFSpatialColumnMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala similarity index 65% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/RFSpatialColumnMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala index af744f5f4..4eade42ad 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/RFSpatialColumnMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala @@ -15,56 +15,71 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.{RasterFrame, StandardColumns} -import com.vividsolutions.jts.geom.{Point, Polygon} +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.RasterFrameLayer +import org.locationtech.jts.geom.Point import geotrellis.proj4.LatLng import geotrellis.spark.SpatialKey import geotrellis.spark.tiling.MapKeyTransform import geotrellis.util.MethodExtensions +import geotrellis.vector.Extent import org.apache.spark.sql.Row -import org.apache.spark.sql.functions.{asc, udf ⇒ sparkUdf} +import org.apache.spark.sql.functions.{asc, udf => sparkUdf} import org.apache.spark.sql.types.{DoubleType, StructField, StructType} import org.locationtech.geomesa.curve.Z2SFC +import org.locationtech.rasterframes.StandardColumns /** - * RasterFrame extension methods associated with adding spatially descriptive columns. + * RasterFrameLayer extension methods associated with adding spatially descriptive columns. * * @since 12/15/17 */ -trait RFSpatialColumnMethods extends MethodExtensions[RasterFrame] with StandardColumns { - import Implicits.{WithDataFrameMethods, WithRasterFrameMethods} +trait RFSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with StandardColumns { + import Implicits.{WithDataFrameMethods, WithRasterFrameLayerMethods} import org.locationtech.geomesa.spark.jts._ /** Returns the key-space to map-space coordinate transform. */ def mapTransform: MapKeyTransform = self.tileLayerMetadata.merge.mapTransform - private def keyCol2Bounds: Row ⇒ Polygon = { + private def keyCol2Extent: Row ⇒ Extent = { val transform = self.sparkSession.sparkContext.broadcast(mapTransform) - (r: Row) ⇒ transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1))).jtsGeom + r ⇒ transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1))) } private def keyCol2LatLng: Row ⇒ (Double, Double) = { val transform = self.sparkSession.sparkContext.broadcast(mapTransform) val crs = self.tileLayerMetadata.merge.crs - (r: Row) ⇒ { + r ⇒ { val center = transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1))).center.reproject(crs, LatLng) (center.x, center.y) } } + /** + * Append a column containing the extent of the row's spatial key. + * Coordinates are in native CRS. + * @param colName name of column to append. Defaults to "extent" + * @return updated RasterFrameLayer + */ + def withExtent(colName: String = EXTENT_COLUMN.columnName): RasterFrameLayer = { + val key2Extent = sparkUdf(keyCol2Extent) + self.withColumn(colName, key2Extent(self.spatialKeyColumn)).certify + } + /** * Append a column containing the bounds of the row's spatial key. * Coordinates are in native CRS. - * @param colName name of column to append. Defaults to "bounds" - * @return updated RasterFrame + * @param colName name of column to append. Defaults to "geometry" + * @return updated RasterFrameLayer */ - def withBounds(colName: String = BOUNDS_COLUMN.columnName): RasterFrame = { - val key2Bounds = sparkUdf(keyCol2Bounds) + def withGeometry(colName: String = GEOMETRY_COLUMN.columnName): RasterFrameLayer = { + val key2Bounds = sparkUdf(keyCol2Extent andThen (_.jtsGeom)) self.withColumn(colName, key2Bounds(self.spatialKeyColumn)).certify } @@ -72,10 +87,10 @@ trait RFSpatialColumnMethods extends MethodExtensions[RasterFrame] with Standard * Append a column containing the center of the row's spatial key. * Coordinate is in native CRS. * @param colName name of column to append. Defaults to "center" - * @return updated RasterFrame + * @return updated RasterFrameLayer */ - def withCenter(colName: String = CENTER_COLUMN.columnName): RasterFrame = { - val key2Center = sparkUdf(keyCol2Bounds andThen (_.getCentroid)) + def withCenter(colName: String = CENTER_COLUMN.columnName): RasterFrameLayer = { + val key2Center = sparkUdf(keyCol2Extent andThen (_.center.jtsGeom)) self.withColumn(colName, key2Center(self.spatialKeyColumn).as[Point]).certify } @@ -83,9 +98,9 @@ trait RFSpatialColumnMethods extends MethodExtensions[RasterFrame] with Standard * Append a column containing the center of the row's spatial key. * Coordinate is in (longitude, latitude) (EPSG:4326). * @param colName name of column to append. Defaults to "center" - * @return updated RasterFrame + * @return updated RasterFrameLayer */ - def withCenterLatLng(colName: String = "center"): RasterFrame = { + def withCenterLatLng(colName: String = "center"): RasterFrameLayer = { val key2Center = sparkUdf(keyCol2LatLng) self.withColumn(colName, key2Center(self.spatialKeyColumn).cast(RFSpatialColumnMethods.LngLatStructType)).certify } @@ -94,9 +109,9 @@ trait RFSpatialColumnMethods extends MethodExtensions[RasterFrame] with Standard * Appends a spatial index column * @param colName name of new column to create. Defaults to `index` * @param applyOrdering if true, adds `.orderBy(asc(colName))` to result. Defaults to `true` - * @return RasterFrame with index column. + * @return RasterFrameLayer with index column. */ - def withSpatialIndex(colName: String = SPATIAL_INDEX_COLUMN.columnName, applyOrdering: Boolean = true): RasterFrame = { + def withSpatialIndex(colName: String = SPATIAL_INDEX_COLUMN.columnName, applyOrdering: Boolean = true): RasterFrameLayer = { val zindex = sparkUdf(keyCol2LatLng andThen (p ⇒ Z2SFC.index(p._1, p._2).z)) self.withColumn(colName, zindex(self.spatialKeyColumn)) match { case rf if applyOrdering ⇒ rf.orderBy(asc(colName)).certify diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala similarity index 79% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala index e83e55fd3..28f2839ed 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala @@ -1,54 +1,64 @@ /* + * This software is licensed under the Apache 2 license, quoted below. + * * Copyright 2017 Astraea, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * [http://www.apache.org/licenses/LICENSE-2.0] * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions import java.time.ZonedDateTime -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.{MetadataKeys, RasterFrame} +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.RasterFrameLayer import geotrellis.proj4.CRS import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} import geotrellis.raster.{MultibandTile, ProjectedRaster, Tile, TileLayout} import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.spark.tiling.{LayoutDefinition, Tiler} -import geotrellis.util.{LazyLogging, MethodExtensions} +import geotrellis.util.MethodExtensions import geotrellis.vector.ProjectedExtent import org.apache.spark.annotation.Experimental import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.{Metadata, TimestampType} import spray.json._ -import astraea.spark.rasterframes.encoders.StandardEncoders._ -import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ +import org.locationtech.rasterframes.encoders.StandardEncoders._ +import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ +import com.typesafe.scalalogging.LazyLogging +import org.locationtech.rasterframes.MetadataKeys +import org.locationtech.rasterframes.tiles.ShowableTile + import scala.reflect.runtime.universe._ /** - * Extension methods on [[RasterFrame]] type. - * @since 7/18/17 + * Extension methods on [[RasterFrameLayer]] type. + * + * @since 7/18/17 */ -trait RasterFrameMethods extends MethodExtensions[RasterFrame] +trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer] with RFSpatialColumnMethods with MetadataKeys with LazyLogging { - import Implicits.{WithDataFrameMethods, WithRasterFrameMethods} + import Implicits.{WithDataFrameMethods, WithRasterFrameLayerMethods} /** - * A convenience over `DataFrame.withColumnRenamed` whereby the `RasterFrame` type is maintained. + * A convenience over `DataFrame.withColumnRenamed` whereby the `RasterFrameLayer` type is maintained. */ - def withRFColumnRenamed(existingName: String, newName: String): RasterFrame = + def withRFColumnRenamed(existingName: String, newName: String): RasterFrameLayer = (self: DataFrame).withColumnRenamed(existingName, newName).certify /** Get the spatial column. */ @@ -66,7 +76,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] def tileLayerMetadata: Either[TileLayerMetadata[SpatialKey], TileLayerMetadata[SpaceTimeKey]] = { val spatialMD = self.findSpatialKeyField .map(_.metadata) - .getOrElse(throw new IllegalArgumentException(s"RasterFrame operation requsted on non-RasterFrame: $self")) + .getOrElse(throw new IllegalArgumentException(s"RasterFrameLayer operation requsted on non-RasterFrameLayer: $self")) if (self.findTemporalKeyField.nonEmpty) Right(extract[TileLayerMetadata[SpaceTimeKey]](CONTEXT_METADATA_KEY)(spatialMD)) @@ -74,12 +84,12 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] Left(extract[TileLayerMetadata[SpatialKey]](CONTEXT_METADATA_KEY)(spatialMD)) } - /** Get the CRS covering the RasterFrame. */ + /** Get the CRS covering the RasterFrameLayer. */ def crs: CRS = tileLayerMetadata.fold(_.crs, _.crs) - /** Add a temporal key to the RasterFrame, assigning the same temporal key to all rows. */ - def addTemporalComponent(value: TemporalKey): RasterFrame = { - require(self.temporalKeyColumn.isEmpty, "RasterFrame already has a temporal component") + /** Add a temporal key to the RasterFrameLayer, assigning the same temporal key to all rows. */ + def addTemporalComponent(value: TemporalKey): RasterFrameLayer = { + require(self.temporalKeyColumn.isEmpty, "RasterFrameLayer already has a temporal component") val tlm = tileLayerMetadata.left.get val newBounds: Bounds[SpaceTimeKey] = tlm.bounds.flatMap[SpaceTimeKey] { @@ -101,14 +111,14 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] } /** Create a temporal key from the given time and assign it as thea temporal key for all rows. */ - def addTemporalComponent(value: ZonedDateTime): RasterFrame = addTemporalComponent(TemporalKey(value)) + def addTemporalComponent(value: ZonedDateTime): RasterFrameLayer = addTemporalComponent(TemporalKey(value)) /** * Append a column containing the temporal key rendered as a TimeStamp. * @param colName name of column to add - * @return updated RasterFrame + * @return updated RasterFrameLayer */ - def withTimestamp(colName: String = TIMESTAMP_COLUMN.columnName): RasterFrame = { + def withTimestamp(colName: String = TIMESTAMP_COLUMN.columnName): RasterFrameLayer = { self.withColumn(colName, (TEMPORAL_KEY_COLUMN.getField("instant").as[Long] / 1000).cast(TimestampType)) .certify } @@ -124,7 +134,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] * @param joinType One of: `inner`, `outer`, `left_outer`, `right_outer`, `leftsemi`. */ @Experimental - def spatialJoin(right: RasterFrame, joinType: String = "inner"): RasterFrame = { + def spatialJoin(right: RasterFrameLayer, joinType: String = "inner"): RasterFrameLayer = { val left = self val leftMetadata = left.tileLayerMetadata.merge @@ -137,7 +147,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] ) } - def updateNames(rf: RasterFrame, + def updateNames(rf: RasterFrameLayer, prefix: String, sk: TypedColumn[Any, SpatialKey], tk: Option[TypedColumn[Any, TemporalKey]]) = { @@ -179,7 +189,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] /** * Performs a full RDD scans of the key column for the data extent, and updates the [[TileLayerMetadata]] data extent to match. */ - def clipLayerExtent: RasterFrame = { + def clipLayerExtent: RasterFrameLayer = { val metadata = tileLayerMetadata val extent = metadata.merge.extent val layout = metadata.merge.layout @@ -213,17 +223,36 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] } /** - * Convert a single tile column from RasterFrame to a GeoTrellis [[TileLayerRDD]] + * Convert a single tile column from RasterFrameLayer to a GeoTrellis [[TileLayerRDD]] * @param tileCol column with tiles to be the */ def toTileLayerRDD(tileCol: Column): Either[TileLayerRDD[SpatialKey], TileLayerRDD[SpaceTimeKey]] = tileLayerMetadata.fold( - tlm ⇒ Left(ContextRDD(self.select(self.spatialKeyColumn, tileCol.as[Tile]).rdd, tlm)), + tlm ⇒ { + val rdd = self.select(self.spatialKeyColumn, tileCol.as[Tile]) + .rdd + .map { + // Wrapped tiles can break GeoTrellis Avro code. + case (sk, wrapped: ShowableTile) => (sk, wrapped.delegate) + case o => o + } + + Left(ContextRDD(rdd, tlm)) + }, tlm ⇒ { val rdd = self .select(self.spatialKeyColumn, self.temporalKeyColumn.get, tileCol.as[Tile]) .rdd - .map { case (sk, tk, v) ⇒ (SpaceTimeKey(sk, tk), v) } + .map { + case (sk, tk, v) ⇒ + val tile = v match { + // Wrapped tiles can break GeoTrellis Avro code. + case wrapped: ShowableTile => wrapped.delegate + case o => o + } + + (SpaceTimeKey(sk, tk), tile) + } Right(ContextRDD(rdd, tlm)) } ) @@ -259,7 +288,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] private[rasterframes] def extract[M: JsonFormat](metadataKey: String)(md: Metadata) = md.getMetadata(metadataKey).json.parseJson.convertTo[M] - /** Convert the tiles in the RasterFrame into a single raster. For RasterFrames keyed with temporal keys, they + /** Convert the tiles in the RasterFrameLayer into a single raster. For RasterFrames keyed with temporal keys, they * will be merge undeterministically. */ def toRaster(tileCol: Column, rasterCols: Int, @@ -294,7 +323,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame] ProjectedRaster(croppedTile.tile, md.extent, md.crs) } - /** Convert the Red, Green & Blue assigned tiles in the RasterFrame into a single color composite raster. + /** Convert the Red, Green & Blue assigned tiles in the RasterFrameLayer into a single color composite raster. * For RasterFrames keyed with temporal keys, they will be merged underterministically. */ def toMultibandRaster( tileCols: Seq[Column], diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala new file mode 100644 index 000000000..e0cec7a8c --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala @@ -0,0 +1,99 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions +import org.apache.spark.sql._ +import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.functions.reproject_and_merge +import org.locationtech.rasterframes.util._ + +import scala.util.Random + +object RasterJoin { + + def apply(left: DataFrame, right: DataFrame): DataFrame = { + val df = apply(left, right, left("extent"), left("crs"), right("extent"), right("crs")) + df.drop(right("extent")).drop(right("crs")) + } + + def apply(left: DataFrame, right: DataFrame, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = { + val leftGeom = st_geometry(leftExtent) + val rightGeomReproj = st_reproject(st_geometry(rightExtent), rightCRS, leftCRS) + val joinExpr = st_intersects(leftGeom, rightGeomReproj) + apply(left, right, joinExpr, leftExtent, leftCRS, rightExtent, rightCRS) + } + + def apply(left: DataFrame, right: DataFrame, joinExprs: Column, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = { + // Convert resolved column into a symbolic one. + def unresolved(c: Column): Column = col(c.columnName) + + // Unique id for temporary columns + val id = Random.alphanumeric.take(5).mkString("_", "", "_") + + // Post aggregation left extent. We preserve the original name. + val leftExtent2 = leftExtent.columnName + // Post aggregation left crs. We preserve the original name. + val leftCRS2 = leftCRS.columnName + // Post aggregation right extent. We create a new name. + val rightExtent2 = id + "extent" + // Post aggregation right crs. We create a new name. + val rightCRS2 = id + "crs" + + + // Gathering up various expressions we'll use to construct the result. + // After joining We will be doing a groupBy the LHS. We have to define the aggregations to perform after the groupBy. + // On the LHS we just want the first thing (subsequent ones should be identical. + val leftAggCols = left.columns.map(s => first(left(s), true) as s) + // On the RHS we collect result as a list. + val rightAggCtx = Seq(collect_list(rightExtent) as rightExtent2, collect_list(rightCRS) as rightCRS2) + val rightAggTiles = right.tileColumns.map(c => collect_list(c) as c.columnName) + val rightAggOther = right.notTileColumns + .filter(n => n.columnName != rightExtent.columnName && n.columnName != rightCRS.columnName) + .map(c => collect_list(c) as (c.columnName + "_agg")) + val aggCols = leftAggCols ++ rightAggTiles ++ rightAggCtx ++ rightAggOther + + // After the aggregation we take all the tiles we've collected and resample + merge + // into LHS extent/CRS. + // Use a representative tile from the left for the tile dimensions + val leftTile = left.tileColumns.headOption.getOrElse(throw new IllegalArgumentException("Need at least one target tile on LHS")) + val reprojCols = rightAggTiles.map(t => reproject_and_merge( + col(leftExtent2), col(leftCRS2), col(t.columnName), col(rightExtent2), col(rightCRS2), rf_dimensions(unresolved(leftTile)) + ) as t.columnName) + + val finalCols = leftAggCols.map(unresolved) ++ reprojCols ++ rightAggOther.map(unresolved) + + // Here's the meat: + left + // 1. Add a unique ID to each LHS row for subequent grouping. + .withColumn(id, monotonically_increasing_id()) + // 2. Perform the left-outer join + .join(right, joinExprs, joinType = "left") + // 3. Group by the unique ID, reestablishing the LHS count + .groupBy(col(id)) + // 4. Apply aggregation to left and right columns: + // a. LHS just take the first entity + // b. RHS collect all results in a list + .agg(aggCols.head, aggCols.tail: _*) + // 5. Perform merge on RHC tile column collections, pass everything else through. + .select(finalCols: _*) + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala new file mode 100644 index 000000000..c396deaee --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala @@ -0,0 +1,49 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions + +import geotrellis.spark.{SpatialKey, TileLayerMetadata} +import org.apache.spark.sql._ +import org.apache.spark.sql.functions.broadcast +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.util._ +object ReprojectToLayer { + + def apply(df: DataFrame, tlm: TileLayerMetadata[SpatialKey]): RasterFrameLayer = { + // create a destination dataframe with crs and extend columns + // use RasterJoin to do the rest. + val gb = tlm.gridBounds + val crs = tlm.crs + + val gridItems = for { + (col, row) <- gb.coordsIter + sk = SpatialKey(col, row) + e = tlm.mapTransform(sk) + } yield (sk, e, crs) + + val dest = df.sparkSession.createDataFrame(gridItems.toSeq) + .toDF(SPATIAL_KEY_COLUMN.columnName, EXTENT_COLUMN.columnName, CRS_COLUMN.columnName) + val joined = RasterJoin(broadcast(dest), df) + + joined.asLayer(SPATIAL_KEY_COLUMN, tlm) + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/SQLContextMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/SQLContextMethods.scala similarity index 84% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/SQLContextMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/SQLContextMethods.scala index bcd2b31c4..4a6df34cc 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/SQLContextMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/SQLContextMethods.scala @@ -15,12 +15,14 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions import geotrellis.util.MethodExtensions -import org.apache.spark.sql.{SQLContext, rf} +import org.apache.spark.sql.SQLContext /** @@ -30,7 +32,7 @@ import org.apache.spark.sql.{SQLContext, rf} */ trait SQLContextMethods extends MethodExtensions[SQLContext] { def withRasterFrames: SQLContext = { - astraea.spark.rasterframes.initRF(self) + org.locationtech.rasterframes.initRF(self) self } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala new file mode 100644 index 000000000..833ba80e3 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala @@ -0,0 +1,59 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.extensions + +import geotrellis.proj4.CRS +import geotrellis.raster.io.geotiff.SinglebandGeoTiff +import geotrellis.util.MethodExtensions +import geotrellis.vector.Extent +import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.{DataFrame, Row, SparkSession} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.TileDimensions + +trait SinglebandGeoTiffMethods extends MethodExtensions[SinglebandGeoTiff] { + def toDF(dims: TileDimensions = NOMINAL_TILE_DIMS)(implicit spark: SparkSession): DataFrame = { + + val segmentLayout = self.imageData.segmentLayout + val re = self.rasterExtent + val crs = self.crs + + val windows = segmentLayout.listWindows(dims.cols, dims.rows) + val subtiles = self.crop(windows) + + val rows = for { + (gridbounds, tile) ← subtiles.toSeq + } yield { + val extent = re.extentFor(gridbounds, false) + Row(extent.toRow, crs.toRow, tile) + } + + val schema = StructType(Seq( + StructField("extent", schemaOf[Extent], false), + StructField("crs", schemaOf[CRS], false), + StructField("tile", TileType, false) + )) + + spark.createDataFrame(spark.sparkContext.makeRDD(rows, 1), schema) + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/SparkSessionMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/SparkSessionMethods.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/extensions/SparkSessionMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/SparkSessionMethods.scala index 9447e812e..a726b4052 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/extensions/SparkSessionMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/SparkSessionMethods.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.extensions +package org.locationtech.rasterframes.extensions import geotrellis.util.MethodExtensions import org.apache.spark.sql.SparkSession @@ -29,7 +31,7 @@ import org.apache.spark.sql.SparkSession */ trait SparkSessionMethods extends MethodExtensions[SparkSession] { def withRasterFrames: SparkSession = { - astraea.spark.rasterframes.initRF(self.sqlContext) + org.locationtech.rasterframes.initRF(self.sqlContext) self } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala similarity index 65% rename from core/src/main/scala/astraea/spark/rasterframes/functions/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/functions/package.scala index 060b08fa3..87894188a 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala @@ -1,28 +1,33 @@ /* + * This software is licensed under the Apache 2 license, quoted below. + * * Copyright 2017 Astraea, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * [http://www.apache.org/licenses/LICENSE-2.0] * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes - -import astraea.spark.rasterframes.expressions.aggstats._ -import astraea.spark.rasterframes.jts.ReprojectionTransformer -import astraea.spark.rasterframes.util.CRSParser -import com.vividsolutions.jts.geom.Geometry -import geotrellis.raster.mapalgebra.local._ +package org.locationtech.rasterframes +import geotrellis.proj4.CRS +import geotrellis.raster.reproject.Reproject import geotrellis.raster.{Tile, _} import geotrellis.vector.Extent -import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.functions.udf +import org.apache.spark.sql.{Row, SQLContext} +import org.locationtech.jts.geom.Geometry +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.TileDimensions /** * Module utils. @@ -66,6 +71,10 @@ package object functions { } } + private[rasterframes] val arrayToTile: (Array[_], Int, Int) ⇒ Tile = (a, cols, rows) ⇒ { + arrayToTile(cols, rows).apply(a) + } + /** Set the tile's no-data value. */ private[rasterframes] def withNoData(nodata: Double) = safeEval[Tile, Tile](_.withNoData(Some(nodata))) @@ -84,6 +93,8 @@ package object functions { } } + + /** Alias for constant tiles of zero */ private[rasterframes] val tileZeros: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒ makeConstantTile(0, cols, rows, cellTypeName) @@ -92,6 +103,36 @@ package object functions { private[rasterframes] val tileOnes: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒ makeConstantTile(1, cols, rows, cellTypeName) + val reproject_and_merge_f: (Row, Row, Seq[Tile], Seq[Row], Seq[Row], Row) => Tile = (leftExtentEnc: Row, leftCRSEnc: Row, tiles: Seq[Tile], rightExtentEnc: Seq[Row], rightCRSEnc: Seq[Row], leftDimsEnc: Row) => { + if (tiles.isEmpty) null + else { + require(tiles.length == rightExtentEnc.length && tiles.length == rightCRSEnc.length, "size mismatch") + + val leftExtent = leftExtentEnc.to[Extent] + val leftDims = leftDimsEnc.to[TileDimensions] + val leftCRS = leftCRSEnc.to[CRS] + val rightExtents = rightExtentEnc.map(_.to[Extent]) + val rightCRSs = rightCRSEnc.map(_.to[CRS]) + + val cellType = tiles.map(_.cellType).reduceOption(_ union _).getOrElse(tiles.head.cellType) + + // TODO: how to allow control over... expression? + val projOpts = Reproject.Options.DEFAULT + val dest: Tile = ArrayTile.empty(cellType, leftDims.cols, leftDims.rows) + //is there a GT function to do all this? + tiles.zip(rightExtents).zip(rightCRSs).map { + case ((tile, extent), crs) => + tile.reproject(extent, crs, leftCRS, projOpts) + }.foldLeft(dest)((d, t) => + d.merge(leftExtent, t.extent, t.tile, projOpts.method) + ) + } + } + + // NB: Don't be tempted to make this a `val`. Spark will barf if `withRasterFrames` hasn't been called first. + def reproject_and_merge = udf(reproject_and_merge_f) + .withName("reproject_and_merge") + private[rasterframes] val cellTypes: () ⇒ Seq[String] = () ⇒ Seq( @@ -125,24 +166,12 @@ package object functions { } } - /** Reporjects a geometry column from one CRS to another, where CRS are defined in Proj4 format. */ - private[rasterframes] val reprojectGeometryCRSName: (Geometry, String, String) ⇒ Geometry = - (sourceGeom, srcName, dstName) ⇒ { - val src = CRSParser(srcName) - val dst = CRSParser(dstName) - val trans = new ReprojectionTransformer(src, dst) - trans.transform(sourceGeom) - } - def register(sqlContext: SQLContext): Unit = { - sqlContext.udf.register("rf_make_constant_tile", makeConstantTile) - sqlContext.udf.register("rf_tile_zeros", tileZeros) - sqlContext.udf.register("rf_tile_ones", tileOnes) - + sqlContext.udf.register("rf_make_zeros_tile", tileZeros) + sqlContext.udf.register("rf_make_ones_tile", tileOnes) sqlContext.udf.register("rf_cell_types", cellTypes) sqlContext.udf.register("rf_rasterize", rasterize) - - sqlContext.udf.register("rf_reproject_geometry", reprojectGeometryCRSName) + sqlContext.udf.register("rf_array_to_tile", arrayToTile) } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala b/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala rename to core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala index e257ebfa5..358fdc258 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala @@ -15,21 +15,23 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.jts +package org.locationtech.rasterframes.jts import java.sql.{Date, Timestamp} import java.time.{LocalDate, ZonedDateTime} -import astraea.spark.rasterframes.expressions.SpatialRelation.{Contains, Intersects} -import com.vividsolutions.jts.geom._ +import org.locationtech.rasterframes.expressions.SpatialRelation.{Contains, Intersects} +import org.locationtech.jts.geom._ import geotrellis.util.MethodExtensions import geotrellis.vector.{Point ⇒ gtPoint} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.functions._ import org.locationtech.geomesa.spark.jts.DataFrameFunctions.SpatialConstructors -import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ +import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ /** * Extension methods on typed columns allowing for DSL-like queries over JTS types. diff --git a/core/src/main/scala/astraea/spark/rasterframes/jts/ReprojectionTransformer.scala b/core/src/main/scala/org/locationtech/rasterframes/jts/ReprojectionTransformer.scala similarity index 85% rename from core/src/main/scala/astraea/spark/rasterframes/jts/ReprojectionTransformer.scala rename to core/src/main/scala/org/locationtech/rasterframes/jts/ReprojectionTransformer.scala index 1d583c739..c4751cb3c 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/jts/ReprojectionTransformer.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/jts/ReprojectionTransformer.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,13 +15,14 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.jts +package org.locationtech.rasterframes.jts -import com.vividsolutions.jts.geom.{CoordinateSequence, Geometry} -import com.vividsolutions.jts.geom.util.GeometryTransformer +import org.locationtech.jts.geom.{CoordinateSequence, Geometry} +import org.locationtech.jts.geom.util.GeometryTransformer import geotrellis.proj4.CRS /** diff --git a/core/src/main/scala/astraea/spark/rasterframes/ml/NoDataFilter.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala similarity index 89% rename from core/src/main/scala/astraea/spark/rasterframes/ml/NoDataFilter.scala rename to core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala index dfe9499a3..5cd9e780e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ml/NoDataFilter.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala @@ -15,17 +15,22 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.ml +package org.locationtech.rasterframes.ml -import astraea.spark.rasterframes.ml.Parameters.HasInputCols +import org.locationtech.rasterframes.ml.Parameters.HasInputCols import org.apache.spark.ml.Transformer import org.apache.spark.ml.param.ParamMap import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable} import org.apache.spark.sql.Dataset import org.apache.spark.sql.types.StructType import java.util.ArrayList + +import org.locationtech.rasterframes.ml.Parameters.HasInputCols + import scala.collection.JavaConversions._ /** diff --git a/core/src/main/scala/astraea/spark/rasterframes/ml/Parameters.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/ml/Parameters.scala rename to core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala index 4bc2fd476..4d273a7f9 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ml/Parameters.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.ml +package org.locationtech.rasterframes.ml import org.apache.spark.ml.param.{Params, StringArrayParam} diff --git a/core/src/main/scala/astraea/spark/rasterframes/ml/TileColumnSupport.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/TileColumnSupport.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/ml/TileColumnSupport.scala rename to core/src/main/scala/org/locationtech/rasterframes/ml/TileColumnSupport.scala index eccc8f00e..d261f7e91 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ml/TileColumnSupport.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ml/TileColumnSupport.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.ml +package org.locationtech.rasterframes.ml import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.{StructField, StructType} diff --git a/core/src/main/scala/astraea/spark/rasterframes/ml/TileExploder.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/ml/TileExploder.scala rename to core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala index d52b82d35..38f978231 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ml/TileExploder.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala @@ -15,18 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.ml +package org.locationtech.rasterframes.ml -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import org.apache.spark.ml.Transformer import org.apache.spark.ml.param.ParamMap import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable} import org.apache.spark.sql.Dataset import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.util._ /** * SparkML Transformer for expanding tiles into single cell rows with @@ -56,7 +58,7 @@ class TileExploder(override val uid: String) extends Transformer val (tiles, nonTiles) = selectTileAndNonTileFields(dataset.schema) val tileCols = tiles.map(f ⇒ col(f.name)) val nonTileCols = nonTiles.map(f ⇒ col(f.name)) - val exploder = explode_tiles(tileCols: _*) + val exploder = rf_explode_tiles(tileCols: _*) dataset.select(nonTileCols :+ exploder: _*) } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/model/CellContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala similarity index 70% rename from core/src/main/scala/astraea/spark/rasterframes/model/CellContext.scala rename to core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala index cac2903dd..95a2e1bf0 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/model/CellContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala @@ -19,25 +19,27 @@ * */ -package astraea.spark.rasterframes.model -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +package org.locationtech.rasterframes.model + import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.{ShortType, StructField, StructType} +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import CatalystSerializer._ -case class CellContext(tile_context: TileContext, tile_data_context: TileDataContext, col_index: Short, row_index: Short) +case class CellContext(tileContext: TileContext, tileDataContext: TileDataContext, colIndex: Short, rowIndex: Short) object CellContext { implicit val serializer: CatalystSerializer[CellContext] = new CatalystSerializer[CellContext] { override def schema: StructType = StructType(Seq( - StructField("tile_context", CatalystSerializer[TileContext].schema, false), - StructField("tile_data_context", CatalystSerializer[TileDataContext].schema, false), - StructField("col_index", ShortType, false), - StructField("row_index", ShortType, false) + StructField("tileContext", schemaOf[TileContext], false), + StructField("tileDataContext", schemaOf[TileDataContext], false), + StructField("colIndex", ShortType, false), + StructField("rowIndex", ShortType, false) )) override protected def to[R](t: CellContext, io: CatalystSerializer.CatalystIO[R]): R = io.create( - io.to(t.tile_context), - io.to(t.tile_data_context), - t.col_index, - t.row_index + io.to(t.tileContext), + io.to(t.tileDataContext), + t.colIndex, + t.rowIndex ) override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): CellContext = CellContext( io.get[TileContext](t, 0), diff --git a/core/src/main/scala/astraea/spark/rasterframes/model/Cells.scala b/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala similarity index 61% rename from core/src/main/scala/astraea/spark/rasterframes/model/Cells.scala rename to core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala index acf847e45..1f7ae4d75 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/model/Cells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala @@ -19,42 +19,61 @@ * */ -package astraea.spark.rasterframes.model -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} -import astraea.spark.rasterframes.ref.RasterRef -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile -import geotrellis.raster.{ArrayTile, Tile} +package org.locationtech.rasterframes.model + +import geotrellis.raster.{ArrayTile, ConstantTile, Tile} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.{BinaryType, StructField, StructType} +import org.locationtech.rasterframes +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import org.locationtech.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes.tiles.ShowableTile +import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile /** Represents the union of binary cell datas or a reference to the data.*/ case class Cells(data: Either[Array[Byte], RasterRef]) { def isRef: Boolean = data.isRight + /** Convert cells into either a RasterRefTile or an ArrayTile. */ def toTile(ctx: TileDataContext): Tile = { data.fold( - bytes => ArrayTile.fromBytes(bytes, ctx.cell_type, ctx.dimensions.cols, ctx.dimensions.rows), + bytes => { + val t = ArrayTile.fromBytes(bytes, ctx.cellType, ctx.dimensions.cols, ctx.dimensions.rows) + if (Cells.showableTiles) new ShowableTile(t) + else t + }, ref => RasterRefTile(ref) ) } } object Cells { + private val showableTiles = rasterframes.rfConfig.getBoolean("showable-tiles") /** Extracts the Cells from a Tile. */ def apply(t: Tile): Cells = { t match { + case prt: ConcreteProjectedRasterTile => + apply(prt.t) case ref: RasterRefTile => Cells(Right(ref.rr)) - case o => + case const: ConstantTile => + // Need to expand constant tiles so they can be interpreted properly in catalyst and Python. + // If we don't, the serialization breaks. + Cells(Left(const.toArrayTile().toBytes)) + case o => Cells(Left(o.toBytes)) } } implicit def cellsSerializer: CatalystSerializer[Cells] = new CatalystSerializer[Cells] { - override def schema: StructType = StructType(Seq( - StructField("cells", BinaryType, true), - StructField("ref", CatalystSerializer[RasterRef].schema, true) - )) + override def schema: StructType = + StructType( + Seq( + StructField("cells", BinaryType, true), + StructField("ref", schemaOf[RasterRef], true) + )) override protected def to[R](t: Cells, io: CatalystSerializer.CatalystIO[R]): R = io.create( t.data.left.getOrElse(null), t.data.right.map(rr => io.to(rr)).right.getOrElse(null) diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/FixedRasterExtent.scala b/core/src/main/scala/org/locationtech/rasterframes/model/FixedRasterExtent.scala new file mode 100644 index 000000000..cdce274bb --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/model/FixedRasterExtent.scala @@ -0,0 +1,278 @@ +/* + * Copyright 2016 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.locationtech.rasterframes.model + + +import geotrellis.raster._ +import geotrellis.vector._ + +import scala.math.ceil + +/** + * This class is a copy of the GeoTrellis 2.x `RasterExtent`, + * with [GT 3.0 fixes](https://github.com/locationtech/geotrellis/pull/2953/files) incorporated into the + * new `GridExtent[T]` class. This class should be removed after RasterFrames is upgraded to GT 3.x. + */ +case class FixedRasterExtent( + override val extent: Extent, + override val cellwidth: Double, + override val cellheight: Double, + cols: Int, + rows: Int +) extends GridExtent(extent, cellwidth, cellheight) with Grid { + import FixedRasterExtent._ + + if (cols <= 0) throw GeoAttrsError(s"invalid cols: $cols") + if (rows <= 0) throw GeoAttrsError(s"invalid rows: $rows") + + /** + * Convert map coordinates (x, y) to grid coordinates (col, row). + */ + final def mapToGrid(x: Double, y: Double): (Int, Int) = { + val col = floorWithTolerance((x - extent.xmin) / cellwidth).toInt + val row = floorWithTolerance((extent.ymax - y) / cellheight).toInt + (col, row) + } + + /** + * Convert map coordinate x to grid coordinate column. + */ + final def mapXToGrid(x: Double): Int = floorWithTolerance(mapXToGridDouble(x)).toInt + + /** + * Convert map coordinate x to grid coordinate column. + */ + final def mapXToGridDouble(x: Double): Double = (x - extent.xmin) / cellwidth + + /** + * Convert map coordinate y to grid coordinate row. + */ + final def mapYToGrid(y: Double): Int = floorWithTolerance(mapYToGridDouble(y)).toInt + + /** + * Convert map coordinate y to grid coordinate row. + */ + final def mapYToGridDouble(y: Double): Double = (extent.ymax - y ) / cellheight + + /** + * Convert map coordinate tuple (x, y) to grid coordinates (col, row). + */ + final def mapToGrid(mapCoord: (Double, Double)): (Int, Int) = { + val (x, y) = mapCoord + mapToGrid(x, y) + } + + /** + * Convert a point to grid coordinates (col, row). + */ + final def mapToGrid(p: Point): (Int, Int) = + mapToGrid(p.x, p.y) + + /** + * The map coordinate of a grid cell is the center point. + */ + final def gridToMap(col: Int, row: Int): (Double, Double) = { + val x = col * cellwidth + extent.xmin + (cellwidth / 2) + val y = extent.ymax - (row * cellheight) - (cellheight / 2) + + (x, y) + } + + /** + * For a give column, find the corresponding x-coordinate in the + * grid of the present [[FixedRasterExtent]]. + */ + final def gridColToMap(col: Int): Double = { + col * cellwidth + extent.xmin + (cellwidth / 2) + } + + /** + * For a give row, find the corresponding y-coordinate in the grid + * of the present [[FixedRasterExtent]]. + */ + final def gridRowToMap(row: Int): Double = { + extent.ymax - (row * cellheight) - (cellheight / 2) + } + + /** + * Gets the GridBounds aligned with this FixedRasterExtent that is the + * smallest subgrid of containing all points within the extent. The + * extent is considered inclusive on it's north and west borders, + * exclusive on it's east and south borders. See [[FixedRasterExtent]] + * for a discussion of grid and extent boundary concepts. + * + * The 'clamp' flag determines whether or not to clamp the + * GridBounds to the FixedRasterExtent; defaults to true. If false, + * GridBounds can contain negative values, or values outside of + * this FixedRasterExtent's boundaries. + * + * @param subExtent The extent to get the grid bounds for + * @param clamp A boolean + */ + def gridBoundsFor(subExtent: Extent, clamp: Boolean = true): GridBounds = { + // West and North boundaries are a simple mapToGrid call. + val (colMin, rowMin) = mapToGrid(subExtent.xmin, subExtent.ymax) + + // If South East corner is on grid border lines, we want to still only include + // what is to the West and\or North of the point. However if the border point + // is not directly on a grid division, include the whole row and/or column that + // contains the point. + val colMax = { + val colMaxDouble = mapXToGridDouble(subExtent.xmax) + if(math.abs(colMaxDouble - floorWithTolerance(colMaxDouble)) < FixedRasterExtent.epsilon) colMaxDouble.toInt - 1 + else colMaxDouble.toInt + } + + val rowMax = { + val rowMaxDouble = mapYToGridDouble(subExtent.ymin) + if(math.abs(rowMaxDouble - floorWithTolerance(rowMaxDouble)) < FixedRasterExtent.epsilon) rowMaxDouble.toInt - 1 + else rowMaxDouble.toInt + } + + if(clamp) { + GridBounds(math.min(math.max(colMin, 0), cols - 1), + math.min(math.max(rowMin, 0), rows - 1), + math.min(math.max(colMax, 0), cols - 1), + math.min(math.max(rowMax, 0), rows - 1)) + } else { + GridBounds(colMin, rowMin, colMax, rowMax) + } + } + + /** + * Combine two different [[FixedRasterExtent]]s (which must have the + * same cellsizes). The result is a new extent at the same + * resolution. + */ + def combine (that: FixedRasterExtent): FixedRasterExtent = { + if (cellwidth != that.cellwidth) + throw GeoAttrsError(s"illegal cellwidths: $cellwidth and ${that.cellwidth}") + if (cellheight != that.cellheight) + throw GeoAttrsError(s"illegal cellheights: $cellheight and ${that.cellheight}") + + val newExtent = extent.combine(that.extent) + val newRows = ceil(newExtent.height / cellheight).toInt + val newCols = ceil(newExtent.width / cellwidth).toInt + + FixedRasterExtent(newExtent, cellwidth, cellheight, newCols, newRows) + } + + /** + * Returns a [[RasterExtent]] with the same extent, but a modified + * number of columns and rows based on the given cell height and + * width. + */ + def withResolution(targetCellWidth: Double, targetCellHeight: Double): FixedRasterExtent = { + val newCols = math.ceil((extent.xmax - extent.xmin) / targetCellWidth).toInt + val newRows = math.ceil((extent.ymax - extent.ymin) / targetCellHeight).toInt + FixedRasterExtent(extent, targetCellWidth, targetCellHeight, newCols, newRows) + } + + /** + * Returns a [[FixedRasterExtent]] with the same extent, but a modified + * number of columns and rows based on the given cell height and + * width. + */ + def withResolution(cellSize: CellSize): FixedRasterExtent = + withResolution(cellSize.width, cellSize.height) + + /** + * Returns a [[FixedRasterExtent]] with the same extent and the given + * number of columns and rows. + */ + def withDimensions(targetCols: Int, targetRows: Int): FixedRasterExtent = + FixedRasterExtent(extent, targetCols, targetRows) + + /** + * Adjusts a raster extent so that it can encompass the tile + * layout. Will resample the extent, but keep the resolution, and + * preserve north and west borders + */ + def adjustTo(tileLayout: TileLayout): FixedRasterExtent = { + val totalCols = tileLayout.tileCols * tileLayout.layoutCols + val totalRows = tileLayout.tileRows * tileLayout.layoutRows + + val resampledExtent = Extent(extent.xmin, extent.ymax - (cellheight*totalRows), + extent.xmin + (cellwidth*totalCols), extent.ymax) + + FixedRasterExtent(resampledExtent, cellwidth, cellheight, totalCols, totalRows) + } + + /** + * Returns a new [[FixedRasterExtent]] which represents the GridBounds + * in relation to this FixedRasterExtent. + */ + def rasterExtentFor(gridBounds: GridBounds): FixedRasterExtent = { + val (xminCenter, ymaxCenter) = gridToMap(gridBounds.colMin, gridBounds.rowMin) + val (xmaxCenter, yminCenter) = gridToMap(gridBounds.colMax, gridBounds.rowMax) + val (hcw, hch) = (cellwidth / 2, cellheight / 2) + val e = Extent(xminCenter - hcw, yminCenter - hch, xmaxCenter + hcw, ymaxCenter + hch) + FixedRasterExtent(e, cellwidth, cellheight, gridBounds.width, gridBounds.height) + } +} + +/** + * The companion object for the [[FixedRasterExtent]] type. + */ +object FixedRasterExtent { + final val epsilon = 0.0000001 + + /** + * Create a new [[FixedRasterExtent]] from an Extent, a column, and a + * row. + */ + def apply(extent: Extent, cols: Int, rows: Int): FixedRasterExtent = { + val cw = extent.width / cols + val ch = extent.height / rows + FixedRasterExtent(extent, cw, ch, cols, rows) + } + + /** + * Create a new [[FixedRasterExtent]] from an Extent and a [[CellSize]]. + */ + def apply(extent: Extent, cellSize: CellSize): FixedRasterExtent = { + val cols = (extent.width / cellSize.width).toInt + val rows = (extent.height / cellSize.height).toInt + FixedRasterExtent(extent, cellSize.width, cellSize.height, cols, rows) + } + + /** + * Create a new [[FixedRasterExtent]] from a [[CellGrid]] and an Extent. + */ + def apply(tile: CellGrid, extent: Extent): FixedRasterExtent = + apply(extent, tile.cols, tile.rows) + + /** + * Create a new [[FixedRasterExtent]] from an Extent and a [[CellGrid]]. + */ + def apply(extent: Extent, tile: CellGrid): FixedRasterExtent = + apply(extent, tile.cols, tile.rows) + + + /** + * The same logic is used in QGIS: https://github.com/qgis/QGIS/blob/607664c5a6b47c559ed39892e736322b64b3faa4/src/analysis/raster/qgsalignraster.cpp#L38 + * The search query: https://github.com/qgis/QGIS/search?p=2&q=floor&type=&utf8=%E2%9C%93 + * + * GDAL uses smth like that, however it was a bit hard to track it down: + * https://github.com/OSGeo/gdal/blob/7601a637dfd204948d00f4691c08f02eb7584de5/gdal/frmts/vrt/vrtsources.cpp#L215 + * */ + def floorWithTolerance(value: Double): Double = { + val roundedValue = math.round(value) + if (math.abs(value - roundedValue) < epsilon) roundedValue + else math.floor(value) + } +} + diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala b/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala new file mode 100644 index 000000000..66352e258 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala @@ -0,0 +1,71 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.model + +import LazyCRS.EncodedCRS +import com.github.blemale.scaffeine.Scaffeine +import geotrellis.proj4.CRS +import org.locationtech.proj4j.CoordinateReferenceSystem + +class LazyCRS(val encoded: EncodedCRS) extends CRS { + private lazy val delegate = LazyCRS.cache.get(encoded) + override def proj4jCrs: CoordinateReferenceSystem = delegate.proj4jCrs + override def toProj4String: String = + if (encoded.startsWith("+proj")) encoded + else delegate.toProj4String + + override def equals(o: Any): Boolean = o match { + case l: LazyCRS => + encoded == l.encoded || + toProj4String == l.toProj4String || + super.equals(o) + case c: CRS => + toProj4String == c.toProj4String || + delegate.equals(c) + case _ => false + } +} + +object LazyCRS { + trait ValidatedCRS + type EncodedCRS = String with ValidatedCRS + + @transient + private lazy val mapper: PartialFunction[String, CRS] = { + case e if e.toUpperCase().startsWith("EPSG") => CRS.fromName(e) //not case-sensitive + case p if p.startsWith("+proj") => CRS.fromString(p) // case sensitive + case w if w.toUpperCase().startsWith("GEOGCS") => CRS.fromWKT(w) //only case-sensitive inside double quotes + } + + @transient + private lazy val cache = Scaffeine().build[String, CRS](mapper) + + def apply(crs: CRS): LazyCRS = apply(crs.toProj4String) + + def apply(value: String): LazyCRS = { + if (mapper.isDefinedAt(value)) { + new LazyCRS(value.asInstanceOf[EncodedCRS]) + } + else throw new IllegalArgumentException( + "crs string must be either EPSG code, +proj string, or OGC WKT") + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/model/TileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala similarity index 83% rename from core/src/main/scala/astraea/spark/rasterframes/model/TileContext.scala rename to core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala index f5d49524c..912e1d81e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/model/TileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala @@ -19,14 +19,16 @@ * */ -package astraea.spark.rasterframes.model -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} -import astraea.spark.rasterframes.tiles.ProjectedRasterTile +package org.locationtech.rasterframes.model + import geotrellis.proj4.CRS import geotrellis.raster.Tile import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.{StructField, StructType} +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import org.locationtech.rasterframes.tiles.ProjectedRasterTile case class TileContext(extent: Extent, crs: CRS) { def toProjectRasterTile(t: Tile): ProjectedRasterTile = ProjectedRasterTile(t, extent, crs) @@ -39,8 +41,8 @@ object TileContext { } implicit val serializer: CatalystSerializer[TileContext] = new CatalystSerializer[TileContext] { override def schema: StructType = StructType(Seq( - StructField("extent", CatalystSerializer[Extent].schema, false), - StructField("crs", CatalystSerializer[CRS].schema, false) + StructField("extent", schemaOf[Extent], false), + StructField("crs", schemaOf[CRS], false) )) override protected def to[R](t: TileContext, io: CatalystSerializer.CatalystIO[R]): R = io.create( io.to(t.extent), diff --git a/core/src/main/scala/astraea/spark/rasterframes/model/TileDataContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala similarity index 80% rename from core/src/main/scala/astraea/spark/rasterframes/model/TileDataContext.scala rename to core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala index 121f8b845..9f6bd358f 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/model/TileDataContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala @@ -19,15 +19,16 @@ * */ -package astraea.spark.rasterframes.model -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} -import astraea.spark.rasterframes.encoders.CatalystSerializer._ +package org.locationtech.rasterframes.model + +import org.locationtech.rasterframes.encoders.CatalystSerializer._ import geotrellis.raster.{CellType, Tile} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.{StructField, StructType} +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} /** Encapsulates all information about a tile aside from actual cell values. */ -case class TileDataContext(cell_type: CellType, dimensions: TileDimensions) +case class TileDataContext(cellType: CellType, dimensions: TileDimensions) object TileDataContext { /** Extracts the TileDataContext from a Tile. */ @@ -41,12 +42,12 @@ object TileDataContext { implicit val serializer: CatalystSerializer[TileDataContext] = new CatalystSerializer[TileDataContext] { override def schema: StructType = StructType(Seq( - StructField("cell_type", CatalystSerializer[CellType].schema, false), - StructField("dimensions", CatalystSerializer[TileDimensions].schema, false) + StructField("cellType", schemaOf[CellType], false), + StructField("dimensions", schemaOf[TileDimensions], false) )) override protected def to[R](t: TileDataContext, io: CatalystIO[R]): R = io.create( - io.to(t.cell_type), + io.to(t.cellType), io.to(t.dimensions) ) override protected def from[R](t: R, io: CatalystIO[R]): TileDataContext = TileDataContext( diff --git a/core/src/main/scala/astraea/spark/rasterframes/model/TileDimensions.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala similarity index 87% rename from core/src/main/scala/astraea/spark/rasterframes/model/TileDimensions.scala rename to core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala index 2f7f579ba..e419ac668 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/model/TileDimensions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala @@ -19,13 +19,13 @@ * */ -package astraea.spark.rasterframes.model +package org.locationtech.rasterframes.model -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO import geotrellis.raster.Grid import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.{ShortType, StructField, StructType} +import org.locationtech.rasterframes.encoders.CatalystSerializer /** * Typed wrapper for tile size information. @@ -49,8 +49,8 @@ object TileDimensions { ) override protected def from[R](t: R, io: CatalystIO[R]): TileDimensions = TileDimensions( - io.getShort(t, 0), - io.getShort(t, 1) + io.getShort(t, 0).toInt, + io.getShort(t, 1).toInt ) } diff --git a/core/src/main/scala/astraea/spark/rasterframes/package.scala b/core/src/main/scala/org/locationtech/rasterframes/rasterframes.scala similarity index 65% rename from core/src/main/scala/astraea/spark/rasterframes/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/rasterframes.scala index 7b360ed25..1517e8f0e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rasterframes.scala @@ -1,56 +1,62 @@ /* + * This software is licensed under the Apache 2 license, quoted below. + * * Copyright 2017 Astraea, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * [http://www.apache.org/licenses/LICENSE-2.0] * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark - -import astraea.spark.rasterframes.encoders.StandardEncoders -import astraea.spark.rasterframes.util.ZeroSevenCompatibilityKit +package org.locationtech import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.LazyLogging +import geotrellis.raster.isData import geotrellis.raster.{Tile, TileFeature} import geotrellis.spark.{ContextRDD, Metadata, SpaceTimeKey, SpatialKey, TileLayerMetadata} import org.apache.spark.rdd.RDD -import org.apache.spark.sql._ +import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT} +import org.apache.spark.sql.{DataFrame, SQLContext, rf} import org.locationtech.geomesa.spark.jts.DataFrameFunctions +import org.locationtech.rasterframes.encoders.StandardEncoders +import org.locationtech.rasterframes.extensions.Implicits +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.util.ZeroSevenCompatibilityKit import shapeless.tag.@@ -import scala.language.higherKinds import scala.reflect.runtime.universe._ -/** - * Module providing support for RasterFrames. - * `import astraea.spark.rasterframes._`., and then call `rfInit(SQLContext)`. - * - * @since 7/18/17 - */ package object rasterframes extends StandardColumns with RasterFunctions with ZeroSevenCompatibilityKit.RasterFunctions - with rasterframes.extensions.Implicits + with Implicits with rasterframes.jts.Implicits with StandardEncoders with DataFrameFunctions.Library with LazyLogging { + @transient + private[rasterframes] + val rfConfig = ConfigFactory.load().getConfig("rasterframes") + /** The generally expected tile size, as defined by configuration property `rasterframes.nominal-tile-size`.*/ @transient - final val NOMINAL_TILE_SIZE: Int = ConfigFactory.load().getInt("rasterframes.nominal-tile-size") + final val NOMINAL_TILE_SIZE: Int = rfConfig.getInt("nominal-tile-size") + final val NOMINAL_TILE_DIMS: TileDimensions = TileDimensions(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE) /** - * Initialization injection point. Must be called before any RasterFrame + * Initialization injection point. Must be called before any RasterFrameLayer * types are used. */ def initRF(sqlContext: SQLContext): Unit = { @@ -79,15 +85,21 @@ package object rasterframes extends StandardColumns rasterframes.rules.register(sqlContext) } + /** TileUDT type reference. */ + def TileType = new TileUDT() + + /** RasterSourceUDT type reference. */ + def RasterSourceType = new RasterSourceUDT() + /** - * A RasterFrame is just a DataFrame with certain invariants, enforced via the methods that create and transform them: + * A RasterFrameLayer is just a DataFrame with certain invariants, enforced via the methods that create and transform them: * 1. One column is a [[geotrellis.spark.SpatialKey]] or [[geotrellis.spark.SpaceTimeKey]] * 2. One or more columns is a [[Tile]] UDT. * 3. The `TileLayerMetadata` is encoded and attached to the key column. */ - type RasterFrame = DataFrame @@ RasterFrameTag + type RasterFrameLayer = DataFrame @@ RasterFrameTag - /** Tagged type for allowing compiler to help keep track of what has RasterFrame assurances applied to it. */ + /** Tagged type for allowing compiler to help keep track of what has RasterFrameLayer assurances applied to it. */ trait RasterFrameTag type TileFeatureLayerRDD[K, D] = @@ -113,6 +125,8 @@ package object rasterframes extends StandardColumns trait StandardLayerKey[T] extends Serializable { val selfType: TypeTag[T] def isType[R: TypeTag]: Boolean = typeOf[R] =:= selfType.tpe + def coerce[K >: T](tlm: TileLayerMetadata[_]): TileLayerMetadata[K] = + tlm.asInstanceOf[TileLayerMetadata[K]] } object StandardLayerKey { def apply[T: StandardLayerKey]: StandardLayerKey[T] = implicitly @@ -124,4 +138,8 @@ package object rasterframes extends StandardColumns } } + /** Test if a cell value evaluates to true: it is not NoData and it is non-zero */ + def isCellTrue(v: Double): Boolean = isData(v) & v != 0.0 + /** Test if a cell value evaluates to true: it is not NoData and it is non-zero */ + def isCellTrue(v: Int): Boolean = isData(v) & v != 0 } diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala new file mode 100644 index 000000000..c460911a0 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala @@ -0,0 +1,89 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import geotrellis.contrib.vlm.{RasterSource => GTRasterSource} +import geotrellis.proj4.CRS +import geotrellis.raster.io.geotiff.Tags +import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster} +import geotrellis.vector.Extent +import org.locationtech.rasterframes.ref.RasterSource.URIRasterSource + +/** A RasterFrames RasterSource which delegates most operations to a geotrellis-contrib RasterSource */ +abstract class DelegatingRasterSource(source: URI, delegateBuilder: () => GTRasterSource) extends RasterSource with URIRasterSource { + @transient + @volatile + private var _delRef: GTRasterSource = _ + + private def retryableRead[R >: Null](f: GTRasterSource => R): R = synchronized { + try { + if (_delRef == null) + _delRef = delegateBuilder() + f(_delRef) + } + catch { + // On this exeception we attempt to recreate the delegate and read again. + case _: java.nio.BufferUnderflowException => + _delRef = null + val newDel = delegateBuilder() + val result = f(newDel) + _delRef = newDel + result + } + } + + // Bad? + override def equals(obj: Any): Boolean = obj match { + case drs: DelegatingRasterSource => drs.source == source + case _ => false + } + + override def hashCode(): Int = source.hashCode() + + // This helps reduce header reads between serializations + def info: SimpleRasterInfo = SimpleRasterInfo.cache.get(source.toASCIIString, _ => + retryableRead(rs => SimpleRasterInfo(rs)) + ) + + override def cols: Int = info.cols + override def rows: Int = info.rows + override def crs: CRS = info.crs + override def extent: Extent = info.extent + override def cellType: CellType = info.cellType + override def bandCount: Int = info.bandCount + override def tags: Tags = info.tags + + override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + retryableRead(_.readBounds(bounds, bands)) + + override def read(bounds: GridBounds, bands: Seq[Int]): Raster[MultibandTile] = + retryableRead(_.read(bounds, bands) + .getOrElse(throw new IllegalArgumentException(s"Bounds '$bounds' outside of source")) + ) + + override def read(extent: Extent, bands: Seq[Int]): Raster[MultibandTile] = + retryableRead(_.read(extent, bands) + .getOrElse(throw new IllegalArgumentException(s"Extent '$extent' outside of source")) + ) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala new file mode 100644 index 000000000..481155f24 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala @@ -0,0 +1,85 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import com.azavea.gdal.GDALWarp +import com.typesafe.scalalogging.LazyLogging +import geotrellis.contrib.vlm.gdal.{GDALRasterSource => VLMRasterSource} +import geotrellis.proj4.CRS +import geotrellis.raster.io.geotiff.Tags +import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster} +import geotrellis.vector.Extent +import org.locationtech.rasterframes.ref.RasterSource.URIRasterSource + +case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSource { + + @transient + private lazy val gdal: VLMRasterSource = { + val cleaned = source.toASCIIString + .replace("gdal+", "") + .replace("gdal:/", "") + // VSIPath doesn't like single slash "file:/path..." + val tweaked = + if (cleaned.matches("^file:/[^/].*")) + cleaned.replace("file:", "") + else cleaned + + VLMRasterSource(tweaked) + } + + protected def tiffInfo = SimpleRasterInfo.cache.get(source.toASCIIString, _ => SimpleRasterInfo(gdal)) + + override def crs: CRS = tiffInfo.crs + + override def extent: Extent = tiffInfo.extent + + private def metadata = Map.empty[String, String] + + override def cellType: CellType = tiffInfo.cellType + + override def bandCount: Int = tiffInfo.bandCount + + override def cols: Int = tiffInfo.cols + + override def rows: Int = tiffInfo.rows + + override def tags: Tags = Tags(metadata, List.empty) + + override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + gdal.readBounds(bounds, bands) +} + +object GDALRasterSource extends LazyLogging { + def gdalVersion(): String = if (hasGDAL) GDALWarp.get_version_info("--version").trim else "not available" + + @transient + lazy val hasGDAL: Boolean = try { + val _ = new GDALWarp() + true + } catch { + case _: UnsatisfiedLinkError => + logger.warn("GDAL native bindings are not available. Falling back to JVM-based reader for GeoTIFF format.") + false + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala new file mode 100644 index 000000000..3249f1bce --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala @@ -0,0 +1,35 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import geotrellis.spark.io.hadoop.HdfsRangeReader +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.locationtech.rasterframes.ref.RasterSource.{URIRasterSource, URIRasterSourceDebugString} + +case class HadoopGeoTiffRasterSource(source: URI, config: () => Configuration) + extends RangeReaderRasterSource with URIRasterSource with URIRasterSourceDebugString { self => + @transient + protected lazy val rangeReader = HdfsRangeReader(new Path(source.getPath), config()) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala new file mode 100644 index 000000000..3a6a2f5e1 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala @@ -0,0 +1,52 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster, Tile} +import geotrellis.raster.io.geotiff.Tags +import geotrellis.vector.Extent +import org.locationtech.rasterframes.ref.RasterSource.EMPTY_TAGS +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +case class InMemoryRasterSource(tile: Tile, extent: Extent, crs: CRS) extends RasterSource { + def this(prt: ProjectedRasterTile) = this(prt, prt.extent, prt.crs) + + override def rows: Int = tile.rows + + override def cols: Int = tile.cols + + override def cellType: CellType = tile.cellType + + override def bandCount: Int = 1 + + override def tags: Tags = EMPTY_TAGS + + override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + bounds + .map(b => { + val subext = rasterExtent.extentFor(b) + Raster(MultibandTile(tile.crop(b)), subext) + }) + .toIterator + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala new file mode 100644 index 000000000..cedb81c61 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala @@ -0,0 +1,28 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import geotrellis.contrib.vlm.geotiff.GeoTiffRasterSource + +case class JVMGeoTiffRasterSource(source: URI) extends DelegatingRasterSource(source, () => GeoTiffRasterSource(source.toASCIIString)) diff --git a/core/src/main/scala/astraea/spark/rasterframes/ref/ProjectedRasterLike.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/ProjectedRasterLike.scala similarity index 95% rename from core/src/main/scala/astraea/spark/rasterframes/ref/ProjectedRasterLike.scala rename to core/src/main/scala/org/locationtech/rasterframes/ref/ProjectedRasterLike.scala index 7c5b2729d..515c47d12 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ref/ProjectedRasterLike.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/ProjectedRasterLike.scala @@ -19,7 +19,7 @@ * */ -package astraea.spark.rasterframes.ref +package org.locationtech.rasterframes.ref import geotrellis.proj4.CRS import geotrellis.raster.CellGrid diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala new file mode 100644 index 000000000..90df001bd --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala @@ -0,0 +1,64 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import com.typesafe.scalalogging.LazyLogging +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster} +import geotrellis.raster.io.geotiff.Tags +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.util.RangeReader +import geotrellis.vector.Extent +import org.locationtech.rasterframes.util.GeoTiffInfoSupport + +trait RangeReaderRasterSource extends RasterSource with GeoTiffInfoSupport with LazyLogging { + protected def rangeReader: RangeReader + + private def realInfo = + GeoTiffReader.readGeoTiffInfo(rangeReader, streaming = true, withOverviews = false) + + protected lazy val tiffInfo = SimpleRasterInfo(realInfo) + + def crs: CRS = tiffInfo.crs + + def extent: Extent = tiffInfo.extent + + override def cols: Int = tiffInfo.rasterExtent.cols + + override def rows: Int = tiffInfo.rasterExtent.rows + + def cellType: CellType = tiffInfo.cellType + + def bandCount: Int = tiffInfo.bandCount + + override def tags: Tags = tiffInfo.tags + + override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val info = realInfo + val geoTiffTile = GeoTiffReader.geoTiffMultibandTile(info) + val intersectingBounds = bounds.flatMap(_.intersection(this)).toSeq + geoTiffTile.crop(intersectingBounds, bands.toArray).map { + case (gb, tile) => + Raster(tile, rasterExtent.extentFor(gb, clamp = true)) + } + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala similarity index 62% rename from core/src/main/scala/astraea/spark/rasterframes/ref/RasterRef.scala rename to core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index ff176765c..b0aabcc48 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -19,26 +19,26 @@ * */ -package astraea.spark.rasterframes.ref +package org.locationtech.rasterframes.ref -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO -import astraea.spark.rasterframes.tiles.ProjectedRasterTile import com.typesafe.scalalogging.LazyLogging import geotrellis.proj4.CRS -import geotrellis.raster.{CellType, GridBounds, Tile, TileLayout} -import geotrellis.spark.tiling.LayoutDefinition +import geotrellis.raster.{CellType, GridBounds, Tile} import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf.RasterSourceUDT -import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, StructField, StructType} +import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _} +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes.tiles.ProjectedRasterTile /** * A delayed-read projected raster implementation. * * @since 8/21/18 */ -case class RasterRef(source: RasterSource, subextent: Option[Extent]) +case class RasterRef(source: RasterSource, bandIndex: Int, subextent: Option[Extent]) extends ProjectedRasterLike { def crs: CRS = source.crs def extent: Extent = subextent.getOrElse(source.extent) @@ -46,44 +46,24 @@ case class RasterRef(source: RasterSource, subextent: Option[Extent]) def cols: Int = grid.width def rows: Int = grid.height def cellType: CellType = source.cellType - def tile: ProjectedRasterTile = ProjectedRasterTile(realizedTile, extent, crs) + def tile: ProjectedRasterTile = ProjectedRasterTile(RasterRefTile(this), extent, crs) - protected lazy val grid: GridBounds = source.rasterExtent.gridBoundsFor(extent) + protected lazy val grid: GridBounds = source.rasterExtent.gridBoundsFor(extent, true) protected def srcExtent: Extent = extent protected lazy val realizedTile: Tile = { - require(source.bandCount == 1, "Expected singleband tile") - RasterRef.log.trace(s"Fetching $srcExtent from $source") - source.read(srcExtent).left.get.tile - } - - /** Splits this tile into smaller tiles based on the reported - * internal structure of the backing format. May return a single item.*/ - def tileToNative: Seq[RasterRef] = { - val ex = this.extent - this.source.nativeTiling - .filter(_ intersects ex) - .map(e ⇒ RasterRef(this.source, Option(e))) + RasterRef.log.trace(s"Fetching $srcExtent from band $bandIndex of $source") + source.read(srcExtent, Seq(bandIndex)).tile.band(0) } } object RasterRef extends LazyLogging { private val log = logger - - /** Constructor for when data extent cover whole raster. */ - def apply(source: RasterSource): RasterRef = RasterRef(source, None) - - private[rasterframes] - def defaultLayout(rr: RasterRef): LayoutDefinition = - LayoutDefinition(rr.extent, rr.source.nativeLayout - .getOrElse(TileLayout(1, 1, rr.cols, rr.rows)) - ) - case class RasterRefTile(rr: RasterRef) extends ProjectedRasterTile { val extent: Extent = rr.extent val crs: CRS = rr.crs - override val cellType = rr.cellType + override def cellType = rr.cellType override val cols: Int = rr.cols override val rows: Int = rr.rows @@ -98,18 +78,21 @@ object RasterRef extends LazyLogging { val rsType = new RasterSourceUDT() override def schema: StructType = StructType(Seq( StructField("source", rsType, false), - StructField("subextent", CatalystSerializer[Extent].schema, true) + StructField("bandIndex", IntegerType, false), + StructField("subextent", schemaOf[Extent], true) )) override def to[R](t: RasterRef, io: CatalystIO[R]): R = io.create( io.to(t.source)(RasterSourceUDT.rasterSourceSerializer), + t.bandIndex, t.subextent.map(io.to[Extent]).orNull ) override def from[R](row: R, io: CatalystIO[R]): RasterRef = RasterRef( io.get[RasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer), - if (io.isNullAt(row, 1)) None - else Option(io.get[Extent](row, 1)) + io.getInt(row, 1), + if (io.isNullAt(row, 2)) None + else Option(io.get[Extent](row, 2)) ) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala new file mode 100644 index 000000000..0f73f85cf --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala @@ -0,0 +1,181 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import com.github.blemale.scaffeine.Scaffeine +import com.typesafe.scalalogging.LazyLogging +import geotrellis.proj4.CRS +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.Tags +import geotrellis.vector.Extent +import org.apache.hadoop.conf.Configuration +import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.rf.RasterSourceUDT +import org.locationtech.rasterframes.model.{FixedRasterExtent, TileContext, TileDimensions} +import org.locationtech.rasterframes.{NOMINAL_TILE_DIMS, rfConfig} + +import scala.concurrent.duration.Duration + +/** + * Abstraction over fetching geospatial raster data. + * + * @since 8/21/18 + */ +@Experimental +trait RasterSource extends ProjectedRasterLike with Serializable { + import RasterSource._ + + def crs: CRS + + def extent: Extent + + def cellType: CellType + + def bandCount: Int + + def tags: Tags + + def read(bounds: GridBounds, bands: Seq[Int]): Raster[MultibandTile] = + readBounds(Seq(bounds), bands).next() + + def read(extent: Extent, bands: Seq[Int] = SINGLEBAND): Raster[MultibandTile] = + read(rasterExtent.gridBoundsFor(extent, clamp = true), bands) + + def readAll(dims: TileDimensions = NOMINAL_TILE_DIMS, bands: Seq[Int] = SINGLEBAND): Seq[Raster[MultibandTile]] = + layoutBounds(dims).map(read(_, bands)) + + protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] + + def rasterExtent = FixedRasterExtent(extent, cols, rows) + + def cellSize = CellSize(extent, cols, rows) + + def gridExtent = GridExtent(extent, cellSize) + + def tileContext: TileContext = TileContext(extent, crs) + + def layoutExtents(dims: TileDimensions): Seq[Extent] = { + val re = rasterExtent + layoutBounds(dims).map(re.rasterExtentFor).map(_.extent) + } + + def layoutBounds(dims: TileDimensions): Seq[GridBounds] = { + gridBounds.split(dims.cols, dims.rows).toSeq + } +} + +object RasterSource extends LazyLogging { + final val SINGLEBAND = Seq(0) + final val EMPTY_TAGS = Tags(Map.empty, List.empty) + + val cacheTimeout: Duration = Duration.fromNanos(rfConfig.getDuration("raster-source-cache-timeout").toNanos) + + private val rsCache = Scaffeine() + .expireAfterAccess(RasterSource.cacheTimeout) + .build[String, RasterSource] + + implicit def rsEncoder: ExpressionEncoder[RasterSource] = { + RasterSourceUDT // Makes sure UDT is registered first + ExpressionEncoder() + } + + def apply(source: URI): RasterSource = + rsCache.get( + source.toASCIIString, _ => source match { + case IsGDAL() => GDALRasterSource(source) + case IsHadoopGeoTiff() => + // TODO: How can we get the active hadoop configuration + // TODO: without having to pass it through? + val config = () => new Configuration() + HadoopGeoTiffRasterSource(source, config) + case IsDefaultGeoTiff() => JVMGeoTiffRasterSource(source) + case s => throw new UnsupportedOperationException(s"Reading '$s' not supported") + } + ) + + object IsGDAL { + + /** Determine if we should prefer GDAL for all types. */ + private val preferGdal: Boolean = org.locationtech.rasterframes.rfConfig.getBoolean("prefer-gdal") + + val gdalOnlyExtensions = Seq(".jp2", ".mrf", ".hdf", ".vrt") + + def gdalOnly(source: URI): Boolean = + if (gdalOnlyExtensions.exists(source.getPath.toLowerCase.endsWith)) { + require(GDALRasterSource.hasGDAL, s"Can only read $source if GDAL is available") + true + } else false + + /** Extractor for determining if a scheme indicates GDAL preference. */ + def unapply(source: URI): Boolean = { + lazy val schemeIsGdal = Option(source.getScheme()) + .exists(_.startsWith("gdal")) + + gdalOnly(source) || ((preferGdal || schemeIsGdal) && GDALRasterSource.hasGDAL) + } + } + + object IsDefaultGeoTiff { + def unapply(source: URI): Boolean = source.getScheme match { + case "file" | "http" | "https" | "s3" => true + case null | "" ⇒ true + case _ => false + } + } + + object IsHadoopGeoTiff { + def unapply(source: URI): Boolean = source.getScheme match { + case "hdfs" | "s3n" | "s3a" | "wasb" | "wasbs" => true + case _ => false + } + } + + trait URIRasterSource { _: RasterSource => + def source: URI + + abstract override def toString: String = { + s"${getClass.getSimpleName}(${source})" + } + } + trait URIRasterSourceDebugString { _: RasterSource with URIRasterSource with Product => + def toDebugString: String = { + val buf = new StringBuilder() + buf.append(productPrefix) + buf.append("(") + buf.append("source=") + buf.append(source.toASCIIString) + buf.append(", size=") + buf.append(size) + buf.append(", dimensions=") + buf.append(dimensions) + buf.append(", crs=") + buf.append(crs) + buf.append(", extent=") + buf.append(extent) + buf.append(")") + buf.toString + } + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala new file mode 100644 index 000000000..1a67822e5 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala @@ -0,0 +1,82 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import com.github.blemale.scaffeine.Scaffeine +import geotrellis.contrib.vlm.geotiff.GeoTiffRasterSource +import geotrellis.contrib.vlm.{RasterSource => GTRasterSource} +import geotrellis.proj4.CRS +import geotrellis.raster.io.geotiff.Tags +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.{CellType, RasterExtent} +import geotrellis.vector.Extent +import org.locationtech.rasterframes.ref.RasterSource.EMPTY_TAGS + +case class SimpleRasterInfo( + cols: Int, + rows: Int, + cellType: CellType, + extent: Extent, + rasterExtent: RasterExtent, + crs: CRS, + tags: Tags, + bandCount: Int, + noDataValue: Option[Double] +) + +object SimpleRasterInfo { + def apply(info: GeoTiffReader.GeoTiffInfo): SimpleRasterInfo = + SimpleRasterInfo( + info.segmentLayout.totalCols, + info.segmentLayout.totalRows, + info.cellType, + info.extent, + info.rasterExtent, + info.crs, + info.tags, + info.bandCount, + info.noDataValue + ) + + def apply(rs: GTRasterSource): SimpleRasterInfo = { + def fetchTags: Tags = rs match { + case gt: GeoTiffRasterSource => gt.tiff.tags + case _ => EMPTY_TAGS + } + + SimpleRasterInfo( + rs.cols, + rs.rows, + rs.cellType, + rs.extent, + rs.rasterExtent, + rs.crs, + fetchTags, + rs.bandCount, + None + ) + } + + lazy val cache = Scaffeine() + //.recordStats() + .build[String, SimpleRasterInfo] +} \ No newline at end of file diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilterPushdownRules.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilterPushdownRules.scala rename to core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala index d61640748..3b3e54d6f 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilterPushdownRules.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.rules +package org.locationtech.rasterframes.rules import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilters.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala similarity index 87% rename from core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilters.scala rename to core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala index 1a1128150..cf731b658 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialFilters.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,12 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.rules +package org.locationtech.rasterframes.rules -import com.vividsolutions.jts.geom.Geometry +import org.locationtech.jts.geom.Geometry import org.apache.spark.sql.sources.Filter /** diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialRelationReceiver.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala similarity index 92% rename from core/src/main/scala/astraea/spark/rasterframes/rules/SpatialRelationReceiver.scala rename to core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala index 36c681d14..403d122ea 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/SpatialRelationReceiver.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.rules +package org.locationtech.rasterframes.rules import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources.{BaseRelation, Filter} diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialUDFSubstitutionRules.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialUDFSubstitutionRules.scala new file mode 100644 index 000000000..d6fea76b0 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialUDFSubstitutionRules.scala @@ -0,0 +1,42 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.rules + +import org.locationtech.rasterframes.expressions.SpatialRelation +import org.apache.spark.sql.catalyst.expressions.ScalaUDF +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +/** + * Swaps out spatial relation UDFs for expression forms. + * + * @since 2/19/18 + */ +object SpatialUDFSubstitutionRules extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = { + plan.transform { + case q: LogicalPlan => q.transformExpressions { + case s: ScalaUDF => SpatialRelation.fromUDF(s).getOrElse(s) + } + } + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/TemporalFilters.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala similarity index 91% rename from core/src/main/scala/astraea/spark/rasterframes/rules/TemporalFilters.scala rename to core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala index 51ea9ddd7..5315b63b7 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/TemporalFilters.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.rules +package org.locationtech.rasterframes.rules import java.sql.{Date, Timestamp} diff --git a/core/src/main/scala/astraea/spark/rasterframes/rules/package.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala similarity index 59% rename from core/src/main/scala/astraea/spark/rasterframes/rules/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/rules/package.scala index ff4755a86..0f028e14e 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/rules/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala @@ -1,4 +1,25 @@ -package astraea.spark.rasterframes +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan diff --git a/core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala similarity index 94% rename from core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala rename to core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala index efc4908db..be3d547a3 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,14 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.stats -import astraea.spark.rasterframes.encoders.StandardEncoders +package org.locationtech.rasterframes.stats + import geotrellis.raster.Tile import geotrellis.raster.histogram.{Histogram => GTHistogram} import org.apache.spark.sql.types._ +import org.locationtech.rasterframes.encoders.StandardEncoders import scala.collection.mutable.{ListBuffer => MutableListBuffer} @@ -160,13 +162,13 @@ object CellHistogram { } else { val h = tile.histogram - h.binCounts().map(p ⇒ Bin(p._1, p._2)) + h.binCounts().map(p ⇒ Bin(p._1.toDouble, p._2)) } CellHistogram(bins) } def apply(hist: GTHistogram[Int]): CellHistogram = { - CellHistogram(hist.binCounts().map(p ⇒ Bin(p._1, p._2))) + CellHistogram(hist.binCounts().map(p ⇒ Bin(p._1.toDouble, p._2))) } def apply(hist: GTHistogram[Double])(implicit ev: DummyImplicit): CellHistogram = { CellHistogram(hist.binCounts().map(p ⇒ Bin(p._1, p._2))) diff --git a/core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala b/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala similarity index 93% rename from core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala rename to core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala index e1ba03b60..ea371666d 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,13 +15,15 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.stats -import astraea.spark.rasterframes.encoders.StandardEncoders +package org.locationtech.rasterframes.stats + import geotrellis.raster.Tile import org.apache.spark.sql.types.StructType +import org.locationtech.rasterframes.encoders.StandardEncoders /** * Container for computed statistics over cells. diff --git a/core/src/main/scala/astraea/spark/rasterframes/stats/LocalCellStatistics.scala b/core/src/main/scala/org/locationtech/rasterframes/stats/LocalCellStatistics.scala similarity index 94% rename from core/src/main/scala/astraea/spark/rasterframes/stats/LocalCellStatistics.scala rename to core/src/main/scala/org/locationtech/rasterframes/stats/LocalCellStatistics.scala index 685722f62..39c0671f8 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/stats/LocalCellStatistics.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/stats/LocalCellStatistics.scala @@ -19,7 +19,8 @@ * */ -package astraea.spark.rasterframes.stats +package org.locationtech.rasterframes.stats + import geotrellis.raster.Tile case class LocalCellStatistics(count: Tile, min: Tile, max: Tile, mean: Tile, variance: Tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/FixedDelegatingTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/FixedDelegatingTile.scala new file mode 100644 index 000000000..52bfa5c1d --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/FixedDelegatingTile.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.tiles +import geotrellis.raster.{ArrayTile, DelegatingTile, Tile} + +/** + * Temporary workaroud for https://github.com/locationtech/geotrellis/issues/2907 + * + * @since 8/22/18 + */ +trait FixedDelegatingTile extends DelegatingTile { + override def combine(r2: Tile)(f: (Int, Int) ⇒ Int): Tile = (delegate, r2) match { + case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combine(r2.toArrayTile())(f) + case _ ⇒ delegate.combine(r2)(f) + } + + override def combineDouble(r2: Tile)(f: (Double, Double) ⇒ Double): Tile = (delegate, r2) match { + case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combineDouble(r2.toArrayTile())(f) + case _ ⇒ delegate.combineDouble(r2)(f) + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/tiles/InternalRowTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala similarity index 86% rename from core/src/main/scala/astraea/spark/rasterframes/tiles/InternalRowTile.scala rename to core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala index 021f0946a..98be22446 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/tiles/InternalRowTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala @@ -19,52 +19,38 @@ * */ -package astraea.spark.rasterframes.tiles +package org.locationtech.rasterframes.tiles import java.nio.ByteBuffer -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO -import astraea.spark.rasterframes.model.{Cells, TileDataContext} +import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO import geotrellis.raster._ import org.apache.spark.sql.catalyst.InternalRow +import org.locationtech.rasterframes.model.{Cells, TileDataContext} /** * Wrapper around a `Tile` encoded in a Catalyst `InternalRow`, for the purpose * of providing compatible semantics over common operations. * - * @groupname COPIES Memory Copying - * @groupdesc COPIES Requires creating an intermediate copy of - * the complete `Tile` contents, and should be avoided. - * * @since 11/29/17 */ -class InternalRowTile(val mem: InternalRow) extends DelegatingTile { +class InternalRowTile(val mem: InternalRow) extends FixedDelegatingTile { import InternalRowTile._ - /** @group COPIES */ - override def toArrayTile(): ArrayTile = realizedTile + override def toArrayTile(): ArrayTile = realizedTile.toArrayTile() - // TODO: We want to reimpliement the delegated methods so that they read directly from tungsten storage - protected lazy val realizedTile: ArrayTile = { - val data = toBytes - if(data.length < cols * rows && cellType.name != "bool") { - val ctile = ConstantTile.fromBytes(data, cellType, cols, rows) - val atile = ctile.toArrayTile() - atile - } - else - ArrayTile.fromBytes(data, cellType, cols, rows) - } + // TODO: We want to reimplement relevant delegated methods so that they read directly from tungsten storage + lazy val realizedTile: Tile = cells.toTile(cellContext) - /** @group COPIES */ protected override def delegate: Tile = realizedTile - private lazy val cellContext: TileDataContext = + private def cellContext: TileDataContext = CatalystIO[InternalRow].get[TileDataContext](mem, 0) + private def cells: Cells = CatalystIO[InternalRow].get[Cells](mem, 1) /** Retrieve the cell type from the internal encoding. */ - override def cellType: CellType = cellContext.cell_type + override def cellType: CellType = cellContext.cellType /** Retrieve the number of columns from the internal encoding. */ override def cols: Int = cellContext.dimensions.cols @@ -74,13 +60,9 @@ class InternalRowTile(val mem: InternalRow) extends DelegatingTile { /** Get the internally encoded tile data cells. */ override lazy val toBytes: Array[Byte] = { - val cellData = CatalystIO[InternalRow] - .get[Cells](mem, 1) - .data - - cellData.left + cells.data.left .getOrElse(throw new IllegalStateException( - "Expected tile cell bytes, but received RasterRef instead: " + cellData.right.get) + "Expected tile cell bytes, but received RasterRef instead: " + cells.data.right.get) ) } @@ -98,12 +80,11 @@ class InternalRowTile(val mem: InternalRow) extends DelegatingTile { } /** Reads the cell value at the given index as an Int. */ - def apply(i: Int): Int = cellReader(i) + def apply(i: Int): Int = cellReader.apply(i) /** Reads the cell value at the given index as a Double. */ def applyDouble(i: Int): Double = cellReader.applyDouble(i) - /** @group COPIES */ def copy = new InternalRowTile(mem.copy) private lazy val cellReader: CellReader = { @@ -132,6 +113,8 @@ class InternalRowTile(val mem: InternalRow) extends DelegatingTile { case _: DoubleCells ⇒ DoubleCellReader(this) } } + + override def toString: String = ShowableTile.show(this) } object InternalRowTile { diff --git a/core/src/main/scala/astraea/spark/rasterframes/tiles/ProjectedRasterTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala similarity index 77% rename from core/src/main/scala/astraea/spark/rasterframes/tiles/ProjectedRasterTile.scala rename to core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala index a9551dd13..92e2d285d 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/tiles/ProjectedRasterTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala @@ -19,32 +19,33 @@ * */ -package astraea.spark.rasterframes.tiles +package org.locationtech.rasterframes.tiles -import astraea.spark.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} -import astraea.spark.rasterframes.encoders.CatalystSerializer.CatalystIO -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.ref.ProjectedRasterLike -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile import geotrellis.proj4.CRS import geotrellis.raster.io.geotiff.SinglebandGeoTiff import geotrellis.raster.{CellType, ProjectedRaster, Tile} import geotrellis.vector.{Extent, ProjectedExtent} -import org.apache.spark.sql.Encoder import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.{StructField, StructType} +import org.locationtech.rasterframes.TileType +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.ref.ProjectedRasterLike +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile /** * A Tile that's also like a ProjectedRaster, with delayed evaluation support. * * @since 9/5/18 */ -trait ProjectedRasterTile extends DelegatingTile with ProjectedRasterLike { +trait ProjectedRasterTile extends FixedDelegatingTile with ProjectedRasterLike { def extent: Extent def crs: CRS def projectedExtent: ProjectedExtent = ProjectedExtent(extent, crs) def projectedRaster: ProjectedRaster[Tile] = ProjectedRaster[Tile](this, extent, crs) + def mapTile(f: Tile => Tile): ProjectedRasterTile = ProjectedRasterTile(f(this), extent, crs) } object ProjectedRasterTile { @@ -58,14 +59,19 @@ object ProjectedRasterTile { case class ConcreteProjectedRasterTile(t: Tile, extent: Extent, crs: CRS) extends ProjectedRasterTile { def delegate: Tile = t + override def convert(cellType: CellType): Tile = ConcreteProjectedRasterTile(t.convert(cellType), extent, crs) - } + override def toString: String = { + val e = s"(${extent.xmin}, ${extent.ymin}, ${extent.xmax}, ${extent.ymax})" + val c = crs.toProj4String + s"[${ShowableTile.show(t)}, $e, $c]" + } + } implicit val serializer: CatalystSerializer[ProjectedRasterTile] = new CatalystSerializer[ProjectedRasterTile] { - val TileType = new TileUDT() override def schema: StructType = StructType(Seq( - StructField("tile_context", CatalystSerializer[TileContext].schema, false), + StructField("tile_context", schemaOf[TileContext], false), StructField("tile", TileType, false)) ) diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala new file mode 100644 index 000000000..00872ff6c --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala @@ -0,0 +1,55 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.tiles +import org.locationtech.rasterframes._ +import geotrellis.raster.Tile + +class ShowableTile(val delegate: Tile) extends FixedDelegatingTile { + override def equals(obj: Any): Boolean = obj match { + case st: ShowableTile => delegate.equals(st.delegate) + case o => delegate.equals(o) + } + override def hashCode(): Int = delegate.hashCode() + override def toString: String = ShowableTile.show(delegate) +} + +object ShowableTile { + private val maxCells = rfConfig.getInt("showable-max-cells") + def show(tile: Tile): String = { + val ct = tile.cellType + val dims = tile.dimensions + + val data = if (tile.cellType.isFloatingPoint) + tile.toArrayDouble() + else tile.toArray() + + val cells = if(tile.size <= maxCells) { + data.mkString("[", ",", "]") + } + else { + val front = data.take(maxCells/2).mkString("[", ",", "") + val back = data.takeRight(maxCells/2).mkString("", ",", "]") + front + ",...," + back + } + s"[${ct.name}, $dims, $cells]" + } +} diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/DataBiasedOp.scala b/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala similarity index 97% rename from core/src/main/scala/astraea/spark/rasterframes/util/DataBiasedOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala index c2e2578a3..83e5fe76c 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/DataBiasedOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala @@ -19,7 +19,7 @@ * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util import geotrellis.raster import geotrellis.raster.isNoData diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/GeoTiffInfoSupport.scala b/core/src/main/scala/org/locationtech/rasterframes/util/GeoTiffInfoSupport.scala similarity index 96% rename from core/src/main/scala/astraea/spark/rasterframes/util/GeoTiffInfoSupport.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/GeoTiffInfoSupport.scala index 724d7eaeb..e24bb8175 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/GeoTiffInfoSupport.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/GeoTiffInfoSupport.scala @@ -19,7 +19,8 @@ * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util + import geotrellis.raster.TileLayout import geotrellis.raster.io.geotiff.reader.GeoTiffReader import geotrellis.raster.io.geotiff.reader.GeoTiffReader.GeoTiffInfo @@ -37,7 +38,7 @@ trait GeoTiffInfoSupport { val MAX_SIZE = 256 private def defaultLayout(cols: Int, rows: Int): TileLayout = { def divs(cells: Int) = { - val layoutDivs = math.ceil(cells / MAX_SIZE.toFloat) + val layoutDivs = math.ceil(cells / MAX_SIZE.toDouble) val tileDivs = math.ceil(cells / layoutDivs) (layoutDivs.toInt, tileDivs.toInt) } diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/KryoSupport.scala b/core/src/main/scala/org/locationtech/rasterframes/util/KryoSupport.scala similarity index 97% rename from core/src/main/scala/astraea/spark/rasterframes/util/KryoSupport.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/KryoSupport.scala index b20aa7851..26754b91d 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/KryoSupport.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/KryoSupport.scala @@ -19,7 +19,7 @@ * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util import java.nio.ByteBuffer diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/MultibandRender.scala b/core/src/main/scala/org/locationtech/rasterframes/util/MultibandRender.scala similarity index 97% rename from core/src/main/scala/astraea/spark/rasterframes/util/MultibandRender.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/MultibandRender.scala index 764d049ee..b576f1e67 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/MultibandRender.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/MultibandRender.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util import geotrellis.raster._ import geotrellis.raster.render.{ColorRamp, Png} diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/RFKryoRegistrator.scala b/core/src/main/scala/org/locationtech/rasterframes/util/RFKryoRegistrator.scala similarity index 72% rename from core/src/main/scala/astraea/spark/rasterframes/util/RFKryoRegistrator.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/RFKryoRegistrator.scala index 58fb62121..8275c6402 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/RFKryoRegistrator.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/RFKryoRegistrator.scala @@ -19,11 +19,11 @@ * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile -import astraea.spark.rasterframes.ref.{RasterRef, RasterSource} -import astraea.spark.rasterframes.ref.RasterSource._ +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes.ref.{DelegatingRasterSource, RasterRef, RasterSource} +import org.locationtech.rasterframes.ref._ import com.esotericsoftware.kryo.Kryo @@ -36,14 +36,15 @@ import com.esotericsoftware.kryo.Kryo class RFKryoRegistrator extends geotrellis.spark.io.kryo.KryoRegistrator { override def registerClasses(kryo: Kryo): Unit = { super.registerClasses(kryo) - kryo.register(classOf[ReadCallback]) kryo.register(classOf[RasterSource]) kryo.register(classOf[RasterRef]) kryo.register(classOf[RasterRefTile]) - kryo.register(classOf[FileGeoTiffRasterSource]) + kryo.register(classOf[DelegatingRasterSource]) + kryo.register(classOf[JVMGeoTiffRasterSource]) + kryo.register(classOf[InMemoryRasterSource]) kryo.register(classOf[HadoopGeoTiffRasterSource]) - kryo.register(classOf[S3GeoTiffRasterSource]) - kryo.register(classOf[HttpGeoTiffRasterSource]) + kryo.register(classOf[GDALRasterSource]) + kryo.register(classOf[SimpleRasterInfo]) kryo.register(classOf[geotrellis.raster.io.geotiff.reader.GeoTiffReader.GeoTiffInfo]) } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala b/core/src/main/scala/org/locationtech/rasterframes/util/SubdivideSupport.scala similarity index 97% rename from core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/SubdivideSupport.scala index 162614651..24ee2ce2d 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/SubdivideSupport.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util import geotrellis.raster.crop.Crop import geotrellis.raster.{CellGrid, TileLayout} diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/ZeroSevenCompatibilityKit.scala b/core/src/main/scala/org/locationtech/rasterframes/util/ZeroSevenCompatibilityKit.scala similarity index 54% rename from core/src/main/scala/astraea/spark/rasterframes/util/ZeroSevenCompatibilityKit.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/ZeroSevenCompatibilityKit.scala index bbb23a282..5826ad09a 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/ZeroSevenCompatibilityKit.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/ZeroSevenCompatibilityKit.scala @@ -19,17 +19,18 @@ * */ -package astraea.spark.rasterframes.util -import astraea.spark.rasterframes.expressions.TileAssembler -import astraea.spark.rasterframes.expressions.accessors._ -import astraea.spark.rasterframes.expressions.aggstats._ -import astraea.spark.rasterframes.expressions.generators._ -import astraea.spark.rasterframes.expressions.localops._ -import astraea.spark.rasterframes.expressions.tilestats._ -import astraea.spark.rasterframes.expressions.transformers._ -import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics} -import astraea.spark.rasterframes.{functions => F} -import com.vividsolutions.jts.geom.Geometry +package org.locationtech.rasterframes.util + +import org.locationtech.rasterframes.expressions.TileAssembler +import org.locationtech.rasterframes.expressions.accessors._ +import org.locationtech.rasterframes.expressions.aggregates._ +import org.locationtech.rasterframes.expressions.generators._ +import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.tilestats._ +import org.locationtech.rasterframes.expressions.transformers._ +import org.locationtech.rasterframes.stats._ +import org.locationtech.rasterframes.{functions => F} +import org.locationtech.jts.geom.Geometry import geotrellis.proj4.CRS import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp import geotrellis.raster.{CellType, Tile} @@ -45,181 +46,181 @@ import org.apache.spark.sql.{Column, SQLContext, TypedColumn, rf} * @since 4/3/17 */ object ZeroSevenCompatibilityKit { - import astraea.spark.rasterframes.encoders.StandardEncoders._ + import org.locationtech.rasterframes.encoders.StandardEncoders._ trait RasterFunctions { - private val delegate = new astraea.spark.rasterframes.RasterFunctions {} + private val delegate = new org.locationtech.rasterframes.RasterFunctions {} // format: off /** Create a row for each cell in Tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def explodeTiles(cols: Column*): Column = delegate.explode_tiles(cols: _*) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def explodeTiles(cols: Column*): Column = delegate.rf_explode_tiles(cols: _*) /** Create a row for each cell in Tile with random sampling and optional seed. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def explodeTilesSample(sampleFraction: Double, seed: Option[Long], cols: Column*): Column = ExplodeTiles(sampleFraction, seed, cols) /** Create a row for each cell in Tile with random sampling (no seed). */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def explodeTilesSample(sampleFraction: Double, cols: Column*): Column = ExplodeTiles(sampleFraction, None, cols) /** Query the number of (cols, rows) in a Tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def tileDimensions(col: Column): Column = GetDimensions(col) @Experimental /** Convert array in `arrayCol` into a Tile of dimensions `cols` and `rows`*/ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def arrayToTile(arrayCol: Column, cols: Int, rows: Int) = withAlias("array_to_tile", arrayCol)( + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def arrayToTile(arrayCol: Column, cols: Int, rows: Int) = withAlias("rf_array_to_tile", arrayCol)( udf[Tile, AnyRef](F.arrayToTile(cols, rows)).apply(arrayCol) ) /** Create a Tile from a column of cell data with location indexes and preform cell conversion. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def assembleTile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int, ct: CellType): TypedColumn[Any, Tile] = convertCellType(TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows)), ct).as(cellData.columnName).as[Tile] /** Create a Tile from a column of cell data with location indexes. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def assembleTile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Column, tileRows: Column): TypedColumn[Any, Tile] = TileAssembler(columnIndex, rowIndex, cellData, tileCols, tileRows) /** Extract the Tile's cell type */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def cellType(col: Column): TypedColumn[Any, CellType] = GetCellType(col) /** Change the Tile's cell type */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def convertCellType(col: Column, cellType: CellType): TypedColumn[Any, Tile] = SetCellType(col, cellType) /** Change the Tile's cell type */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def convertCellType(col: Column, cellTypeName: String): TypedColumn[Any, Tile] = SetCellType(col, cellTypeName) /** Convert a bounding box structure to a Geometry type. Intented to support multiple schemas. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def boundsGeometry(bounds: Column): TypedColumn[Any, Geometry] = BoundsToGeometry(bounds) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def boundsGeometry(bounds: Column): TypedColumn[Any, Geometry] = ExtentToGeometry(bounds) /** Assign a `NoData` value to the Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def withNoData(col: Column, nodata: Double) = withAlias("withNoData", col)( udf[Tile, Tile](F.withNoData(nodata)).apply(col) ).as[Tile] /** Compute the full column aggregate floating point histogram. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def aggHistogram(col: Column): TypedColumn[Any, CellHistogram] = delegate.agg_approx_histogram(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def aggHistogram(col: Column): TypedColumn[Any, CellHistogram] = delegate.rf_agg_approx_histogram(col) /** Compute the full column aggregate floating point statistics. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def aggStats(col: Column): TypedColumn[Any, CellStatistics] = delegate.agg_stats(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def aggStats(col: Column): TypedColumn[Any, CellStatistics] = delegate.rf_agg_stats(col) /** Computes the column aggregate mean. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def aggMean(col: Column) = CellMeanAggregate(col) /** Computes the number of non-NoData cells in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def aggDataCells(col: Column): TypedColumn[Any, Long] = delegate.agg_data_cells(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def aggDataCells(col: Column): TypedColumn[Any, Long] = delegate.rf_agg_data_cells(col) /** Computes the number of NoData cells in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def aggNoDataCells(col: Column): TypedColumn[Any, Long] = delegate.agg_no_data_cells(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def aggNoDataCells(col: Column): TypedColumn[Any, Long] = delegate.rf_agg_no_data_cells(col) /** Compute the Tile-wise mean */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileMean(col: Column): TypedColumn[Any, Double] = delegate.tile_mean(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileMean(col: Column): TypedColumn[Any, Double] = delegate.rf_tile_mean(col) /** Compute the Tile-wise sum */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileSum(col: Column): TypedColumn[Any, Double] = delegate.tile_sum(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileSum(col: Column): TypedColumn[Any, Double] = delegate.rf_tile_sum(col) /** Compute the minimum cell value in tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileMin(col: Column): TypedColumn[Any, Double] = delegate.tile_min(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileMin(col: Column): TypedColumn[Any, Double] = delegate.rf_tile_min(col) /** Compute the maximum cell value in tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileMax(col: Column): TypedColumn[Any, Double] = delegate.tile_max(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileMax(col: Column): TypedColumn[Any, Double] = delegate.rf_tile_max(col) /** Compute TileHistogram of Tile values. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileHistogram(col: Column): TypedColumn[Any, CellHistogram] = delegate.tile_histogram(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileHistogram(col: Column): TypedColumn[Any, CellHistogram] = delegate.rf_tile_histogram(col) /** Compute statistics of Tile values. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def tileStats(col: Column): TypedColumn[Any, CellStatistics] = delegate.tile_stats(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def tileStats(col: Column): TypedColumn[Any, CellStatistics] = delegate.rf_tile_stats(col) /** Counts the number of non-NoData cells per Tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def dataCells(tile: Column): TypedColumn[Any, Long] = delegate.data_cells(tile) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def dataCells(tile: Column): TypedColumn[Any, Long] = delegate.rf_data_cells(tile) /** Counts the number of NoData cells per Tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def noDataCells(tile: Column): TypedColumn[Any, Long] = delegate.no_data_cells(tile) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def noDataCells(tile: Column): TypedColumn[Any, Long] = delegate.rf_no_data_cells(tile) - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def isNoDataTile(tile: Column): TypedColumn[Any, Boolean] = delegate.is_no_data_tile(tile) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def isNoDataTile(tile: Column): TypedColumn[Any, Boolean] = delegate.rf_is_no_data_tile(tile) /** Compute cell-local aggregate descriptive statistics for a column of Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggStats(col: Column): Column = delegate.agg_local_stats(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggStats(col: Column): Column = delegate.rf_agg_local_stats(col) /** Compute the cell-wise/local max operation between Tiles in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggMax(col: Column): TypedColumn[Any, Tile] = delegate.agg_local_max(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggMax(col: Column): TypedColumn[Any, Tile] = delegate.rf_agg_local_max(col) /** Compute the cellwise/local min operation between Tiles in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggMin(col: Column): TypedColumn[Any, Tile] = delegate.agg_local_min(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggMin(col: Column): TypedColumn[Any, Tile] = delegate.rf_agg_local_min(col) /** Compute the cellwise/local mean operation between Tiles in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggMean(col: Column): TypedColumn[Any, Tile] = delegate.agg_local_mean(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggMean(col: Column): TypedColumn[Any, Tile] = delegate.rf_agg_local_mean(col) /** Compute the cellwise/local count of non-NoData cells for all Tiles in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggDataCells(col: Column): TypedColumn[Any, Tile] = delegate.agg_local_data_cells(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggDataCells(col: Column): TypedColumn[Any, Tile] = delegate.rf_agg_local_data_cells(col) /** Compute the cellwise/local count of NoData cells for all Tiles in a column. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAggNoDataCells(col: Column): TypedColumn[Any, Tile] = delegate.agg_local_no_data_cells(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAggNoDataCells(col: Column): TypedColumn[Any, Tile] = delegate.rf_agg_local_no_data_cells(col) /** Cellwise addition between two Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAdd(left: Column, right: Column): Column = delegate.local_add(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAdd(left: Column, right: Column): Column = delegate.rf_local_add(left, right) /** Cellwise addition of a scalar to a tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localAddScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_add(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localAddScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_add(tileCol, value) /** Cellwise subtraction between two Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localSubtract(left: Column, right: Column): Column = delegate.local_subtract(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localSubtract(left: Column, right: Column): Column = delegate.rf_local_subtract(left, right) /** Cellwise subtraction of a scalar from a tile. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localSubtractScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_subtract(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localSubtractScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_subtract(tileCol, value) /** Cellwise multiplication between two Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localMultiply(left: Column, right: Column): Column = delegate.local_multiply(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localMultiply(left: Column, right: Column): Column = delegate.rf_local_multiply(left, right) /** Cellwise multiplication of a tile by a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localMultiplyScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_multiply(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localMultiplyScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_multiply(tileCol, value) /** Cellwise division between two Tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localDivide(left: Column, right: Column): Column = delegate.local_divide(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localDivide(left: Column, right: Column): Column = delegate.rf_local_divide(left, right) /** Cellwise division of a tile by a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localDivideScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_divide(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localDivideScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_divide(tileCol, value) /** Perform an arbitrary GeoTrellis `LocalTileBinaryOp` between two Tile columns. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def localAlgebra(op: LocalTileBinaryOp, left: Column, right: Column): TypedColumn[Any, Tile] = withAlias(opName(op), left, right)( @@ -227,94 +228,94 @@ object ZeroSevenCompatibilityKit { ).as[Tile] /** Compute the normalized difference of two tile columns */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def normalizedDifference(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.normalized_difference(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def normalizedDifference(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.rf_normalized_difference(left, right) /** Constructor for constant tile column */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def makeConstantTile(value: Number, cols: Int, rows: Int, cellType: String): TypedColumn[Any, Tile] = udf(() => F.makeConstantTile(value, cols, rows, cellType)).apply().as(s"constant_$cellType").as[Tile] /** Alias for column of constant tiles of zero */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def tileZeros(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] = udf(() => F.tileZeros(cols, rows, cellType)).apply().as(s"zeros_$cellType").as[Tile] /** Alias for column of constant tiles of one */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def tileOnes(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] = udf(() => F.tileOnes(cols, rows, cellType)).apply().as(s"ones_$cellType").as[Tile] /** Where the mask tile equals the mask value, replace values in the source tile with NODATA */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def maskByValue(sourceTile: Column, maskTile: Column, maskValue: Column): TypedColumn[Any, Tile] = - delegate.mask_by_value(sourceTile, maskTile, maskValue) + delegate.rf_mask_by_value(sourceTile, maskTile, maskValue) /** Where the mask tile DOES NOT contain NODATA, replace values in the source tile with NODATA */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def inverseMask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] = - delegate.inverse_mask(sourceTile, maskTile) + delegate.rf_inverse_mask(sourceTile, maskTile) /** Reproject a column of geometry from one CRS to another. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def reprojectGeometry(sourceGeom: Column, srcCRS: CRS, dstCRS: CRS): TypedColumn[Any, Geometry] = - delegate.reproject_geometry(sourceGeom, srcCRS, dstCRS) + delegate.st_reproject(sourceGeom, srcCRS, dstCRS) /** Render Tile as ASCII string for debugging purposes. */ @Experimental - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def renderAscii(col: Column): TypedColumn[Any, String] = delegate.render_ascii(col) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def renderAscii(col: Column): TypedColumn[Any, String] = delegate.rf_render_ascii(col) /** Cellwise less than value comparison between two tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def localLess(left: Column, right: Column): TypedColumn[Any, Tile] = - delegate.local_less(left, right) + delegate.rf_local_less(left, right) /** Cellwise less than value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localLessScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_less(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localLessScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_less(tileCol, value) /** Cellwise less than or equal to value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localLessEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.local_less_equal(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localLessEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.rf_local_less_equal(left, right) /** Cellwise less than or equal to value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localLessEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_less_equal(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localLessEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_less_equal(tileCol, value) /** Cellwise greater than value comparison between two tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") def localGreater(left: Column, right: Column): TypedColumn[Any, Tile] = - delegate.local_greater(left, right) + delegate.rf_local_greater(left, right) /** Cellwise greater than value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localGreaterScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_greater(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localGreaterScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_greater(tileCol, value) /** Cellwise greater than or equal to value comparison between two tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localGreaterEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.local_greater_equal(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localGreaterEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.rf_local_greater_equal(left, right) /** Cellwise greater than or equal to value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localGreaterEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_greater_equal(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localGreaterEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_greater_equal(tileCol, value) /** Cellwise equal to value comparison between two tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.local_equal(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localEqual(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.rf_local_equal(left, right) /** Cellwise equal to value comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_equal(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_equal(tileCol, value) /** Cellwise inequality comparison between two tiles. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localUnequal(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.local_unequal(left, right) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localUnequal(left: Column, right: Column): TypedColumn[Any, Tile] = delegate.rf_local_unequal(left, right) /** Cellwise inequality comparison between a tile and a scalar. */ - @deprecated("Part of 0.7.x compatility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") - def localUnequalScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.local_unequal(tileCol, value) + @deprecated("Part of 0.7.x compatibility kit, to be removed after 0.8.x. Please use \"snake_case\" variant instead.", "0.8.0") + def localUnequalScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = delegate.rf_local_unequal(tileCol, value) } def register(sqlContext: SQLContext): Unit = { @@ -323,6 +324,8 @@ object ZeroSevenCompatibilityKit { def ub[A, B](f: A => B)(a: Seq[A]): B = f(a.head) /** Binary expression builder builder. */ def bb[A, B](f: (A, A) => B)(a: Seq[A]): B = f(a.head, a.last) + /** Trinary expression builder builder. */ + def tb[A, B](f: (A, A, A) => B)(a: Seq[A]): B = f(a.head, a.tail.head, a.last) // Expression-oriented functions have a different registration scheme // Currently have to register with the `builtin` registry due to Spark data hiding. @@ -331,7 +334,7 @@ object ZeroSevenCompatibilityKit { registry.registerFunc("rf_cellType", ub(GetCellType.apply)) registry.registerFunc("rf_convertCellType", bb(SetCellType.apply)) registry.registerFunc("rf_tileDimensions", ub(GetDimensions.apply)) - registry.registerFunc("rf_boundsGeometry", ub(BoundsToGeometry.apply)) + registry.registerFunc("rf_boundsGeometry", ub(ExtentToGeometry.apply)) registry.registerFunc("rf_localAdd", bb(Add.apply)) registry.registerFunc("rf_localSubtract", bb(Subtract.apply)) registry.registerFunc("rf_localMultiply", bb(Multiply.apply)) @@ -360,11 +363,11 @@ object ZeroSevenCompatibilityKit { registry.registerFunc("rf_localAggMin", ub(LocalTileOpAggregate.LocalMinUDAF.apply)) registry.registerFunc("rf_localAggCount", ub(LocalCountAggregate.LocalDataCellsUDAF.apply)) registry.registerFunc("rf_localAggMean", ub(LocalMeanAggregate.apply)) + registry.registerFunc("rf_reprojectGeometry", tb(ReprojectGeometry.apply)) sqlContext.udf.register("rf_makeConstantTile", F.makeConstantTile) sqlContext.udf.register("rf_tileZeros", F.tileZeros) sqlContext.udf.register("rf_tileOnes", F.tileOnes) sqlContext.udf.register("rf_cellTypes", F.cellTypes) - sqlContext.udf.register("rf_reprojectGeometry", F.reprojectGeometryCRSName) } } diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala b/core/src/main/scala/org/locationtech/rasterframes/util/debug/package.scala similarity index 80% rename from core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/debug/package.scala index 53b4b6aee..e33529b02 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/debug/package.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,12 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.util +package org.locationtech.rasterframes.util -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.proj4.LatLng import geotrellis.vector.{Feature, Geometry} import geotrellis.vector.io.json.JsonFeatureCollection @@ -32,21 +33,21 @@ import spray.json.JsValue * @since 4/6/18 */ package object debug { - implicit class RasterFrameWithDebug(val self: RasterFrame) { + implicit class RasterFrameWithDebug(val self: RasterFrameLayer) { /** Renders the whole schema with metadata as a JSON string. */ def describeFullSchema: String = { self.schema.prettyJson } - /** Renders all the extents in this RasterFrame as GeoJSON in EPSG:4326. This does a full + /** Renders all the extents in this RasterFrameLayer as GeoJSON in EPSG:4326. This does a full * table scan and collects **all** the geometry into the driver, and then converts it into a * Spray JSON data structure. Not performant, and for debugging only. */ def geoJsonExtents: JsValue = { import spray.json.DefaultJsonProtocol._ val features = self - .select(BOUNDS_COLUMN, SPATIAL_KEY_COLUMN) + .select(GEOMETRY_COLUMN, SPATIAL_KEY_COLUMN) .collect() .map{ case (p, s) ⇒ Feature(Geometry(p).reproject(self.crs, LatLng), Map("col" -> s.col, "row" -> s.row)) } diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/package.scala b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala similarity index 79% rename from core/src/main/scala/astraea/spark/rasterframes/util/package.scala rename to core/src/main/scala/org/locationtech/rasterframes/util/package.scala index 02a365cea..e94869986 100644 --- a/core/src/main/scala/astraea/spark/rasterframes/util/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala @@ -15,28 +15,32 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes -import geotrellis.proj4.CRS -import geotrellis.raster -import geotrellis.raster.{CellGrid, Tile, isNoData} +import com.typesafe.scalalogging.Logger import geotrellis.raster.crop.TileCropMethods import geotrellis.raster.io.geotiff.reader.GeoTiffReader import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp import geotrellis.raster.mask.TileMaskMethods import geotrellis.raster.merge.TileMergeMethods import geotrellis.raster.prototype.TilePrototypeMethods +import geotrellis.raster.{CellGrid, Tile, isNoData} import geotrellis.spark.Bounds import geotrellis.spark.tiling.TilerKeyMethods -import geotrellis.util.{ByteReader, GetComponent, LazyLogging} -import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression} +import geotrellis.util.{ByteReader, GetComponent} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpression} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.functions._ import org.apache.spark.sql.rf._ import org.apache.spark.sql.types.StringType -import org.apache.spark.sql.{Column, DataFrame, SQLContext} +import org.apache.spark.sql._ +import org.slf4j.LoggerFactory import spire.syntax.cfor._ import scala.Boolean.box @@ -46,7 +50,10 @@ import scala.Boolean.box * * @since 12/18/17 */ -package object util extends LazyLogging { +package object util { + @transient + protected lazy val logger: Logger = + Logger(LoggerFactory.getLogger("org.locationtech.rasterframes")) import reflect.ClassTag import reflect.runtime.universe._ @@ -71,34 +78,28 @@ package object util extends LazyLogging { type KeyMethodsProvider[K1, K2] = K1 ⇒ TilerKeyMethods[K1, K2] - /** Internal method for slapping the RasterFrame seal of approval on a DataFrame. */ - private[rasterframes] def certifyRasterframe(df: DataFrame): RasterFrame = + /** Internal method for slapping the RasterFrameLayer seal of approval on a DataFrame. */ + private[rasterframes] def certifyRasterframe(df: DataFrame): RasterFrameLayer = shapeless.tag[RasterFrameTag][DataFrame](df) /** Tags output column with a nicer name. */ private[rasterframes] - def withAlias(name: String, inputs: Column*)(output: Column) = { + def withAlias(name: String, inputs: Column*)(output: Column): Column = { val paramNames = inputs.map(_.columnName).mkString(",") output.as(s"$name($paramNames)") } + /** Tags output column with a nicer name, yet strongly typed. */ + private[rasterframes] + def withTypedAlias[T: Encoder](name: String, inputs: Column*)(output: Column): TypedColumn[Any, T] = + withAlias(name, inputs: _*)(output).as[T] + /** Derives and operator name from the implementing object name. */ private[rasterframes] def opName(op: LocalTileBinaryOp) = op.getClass.getSimpleName.replace("$", "").toLowerCase - object CRSParser { - def apply(value: String): CRS = { - value match { - case e if e.toUpperCase().startsWith("EPSG") => CRS.fromName(e) //not case-sensitive - case p if p.startsWith("+proj") => CRS.fromString(p) // case sensitive - case w if w.toUpperCase().startsWith("GEOGCS") => CRS.fromWKT(w) //only case-sensitive inside double quotes - case _ ⇒ throw new IllegalArgumentException("crs string must be either EPSG code, +proj string, or OGC WKT") - } - } - } - implicit class WithCombine[T](left: Option[T]) { def combine[A, R >: A](a: A)(f: (T, A) ⇒ R): R = left.map(f(_, a)).getOrElse(a) def tupleWith[R](right: Option[R]): Option[(T, R)] = left.flatMap(l ⇒ right.map((l, _))) @@ -107,7 +108,9 @@ package object util extends LazyLogging { implicit class ExpressionWithName(val expr: Expression) extends AnyVal { import org.apache.spark.sql.catalyst.expressions.Literal def name: String = expr match { - case n: NamedExpression ⇒ n.name + case n: NamedExpression if n.resolved ⇒ n.name + case UnresolvedAttribute(parts) => parts.mkString("_") + case Alias(_, name) => name case l: Literal if l.dataType == StringType ⇒ String.valueOf(l.value) case o ⇒ o.toString } @@ -181,6 +184,27 @@ package object util extends LazyLogging { } } + implicit class DFWithPrettyPrint(val df: Dataset[_]) extends AnyVal { + def toMarkdown(numRows: Int = 5, truncate: Boolean = false): String = { + import df.sqlContext.implicits._ + val cols = df.columns + val header = cols.mkString("| ", " | ", " |") + "\n" + ("|---" * cols.length) + "|\n" + val stringifiers = cols + .map(c => s"`$c`") + .map(c => df.col(c).cast(StringType)) + .map(c => if (truncate) substring(c, 1, 40) else c) + val cat = concat_ws(" | ", stringifiers: _*) + val body = df + .select(cat).limit(numRows) + .as[String] + .collect() + .map(_.replaceAll("\\[", "\\\\[")) + .map(_.replace('\n', '↩')) + .mkString("| ", " |\n| ", " |") + header + body + } + } + object Shims { // GT 1.2.1 to 2.0.0 def toArrayTile[T <: CellGrid](tile: T): T = diff --git a/core/src/test/resources/B01.jp2 b/core/src/test/resources/B01.jp2 new file mode 100644 index 000000000..18de22f54 Binary files /dev/null and b/core/src/test/resources/B01.jp2 differ diff --git a/core/src/test/resources/L8-B4-Elkton-VA-4326.tiff b/core/src/test/resources/L8-B4-Elkton-VA-4326.tiff new file mode 100644 index 000000000..2bc57e255 Binary files /dev/null and b/core/src/test/resources/L8-B4-Elkton-VA-4326.tiff differ diff --git a/core/src/test/resources/L8-archive.zip b/core/src/test/resources/L8-archive.zip new file mode 100644 index 000000000..93afb4db4 Binary files /dev/null and b/core/src/test/resources/L8-archive.zip differ diff --git a/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.idx b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.idx new file mode 100644 index 000000000..f86df2587 Binary files /dev/null and b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.idx differ diff --git a/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.lrc b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.lrc new file mode 100644 index 000000000..75163d4a2 Binary files /dev/null and b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.lrc differ diff --git a/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf new file mode 100644 index 000000000..8245c4a7e --- /dev/null +++ b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf @@ -0,0 +1,12 @@ + + + + + LERC + + + + + PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]] + + diff --git a/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf.aux.xml b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf.aux.xml new file mode 100644 index 000000000..5a18f6944 --- /dev/null +++ b/core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf.aux.xml @@ -0,0 +1,92 @@ + + + LERC + PIXEL + + + 06121997 + MODIS + MODIS + Terra + Aqua + MODIS + MODIS + Passed + Passed was set as a default value. More algorithm will be developed + 0 + AMBRALS_V4.0R1 + v1.0500m + 15.0 + 463.312716527778 + volume + 2400 + 2400 + Day + Mandatory QA: + 0 = processed, good quality (full BRDF inversions) + 1 = processed, see other QA (magnitude BRDF inversions) + + 6.1 + 150.120692476232 + N + False + 75.0 + 86400 + 43200 + 19.9448109058663, 30.0666177912155, 29.9990071837477, 19.8789125843729 + 127.31379517564, 138.161359988435, 150.130532080915, 138.321766284772 + 1, 2, 3, 4 + HDFEOS_V2.19 + 30 + 10.5067/MODIS/MCD43A4.006 + 10.5067/MODIS/MCD43A4.006 + http://dx.doi.org + http://dx.doi.org + MYD09GA.A2019113.h30v06.006.2019115025936.hdf, MYD09GA.A2019114.h30v06.006.2019117021858.hdf, MYD09GA.A2019115.h30v06.006.2019117044251.hdf, MYD09GA.A2019116.h30v06.006.2019118031111.hdf, MYD09GA.A2019117.h30v06.006.2019119025916.hdf, MYD09GA.A2019118.h30v06.006.2019120030848.hdf, MOD09GA.A2019113.h30v06.006.2019115032521.hdf, MOD09GA.A2019114.h30v06.006.2019116030646.hdf, MOD09GA.A2019115.h30v06.006.2019117050730.hdf, MOD09GA.A2019116.h30v06.006.2019118032616.hdf, MOD09GA.A2019117.h30v06.006.2019119032020.hdf, MOD09GA.A2019118.h30v06.006.2019120032257.hdf, MCD43DB.A2019110.6.h30v06.hdf + MCD43A4.A2019111.h30v06.006.2019120033434.hdf + 6.1.34 + MODIS/Terra+Aqua BRDF/Albedo Nadir BRDF-Adjusted Ref Daily L3 Global - 500m + BRDF_Albedo_Band_Mandatory_Quality_Band1 + 0 + 500m + 29.9999999973059 + 1 + NOT SET + 0 + 0 + 0 + 100 + 0 + 6.0.42 + MODAPS + Linux minion7043 3.10.0-957.5.1.el7.x86_64 #1 SMP Fri Feb 1 14:54:57 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux + 2019-04-30T03:34:48.000Z + 0 + 0 + 99 + 0 + 2019-04-13 + 00:00:00.000000 + 2019-04-28 + 23:59:59.999999 + processed once + further update is anticipated + Not Investigated + See http://landweb.nascom/nasa.gov/cgi-bin/QA_WWW/qaFlagPage.cgi?sat=aqua the product Science Quality status. + 06121997 + MCD43A4 + 19.9999999982039 + 2015 + 51030006 + concatenated flags + 0, 254 + 6 + 6 + 127.701332684185 + 255 + + + BRDF_Albedo_Band_Mandatory_Quality_Band1 + concatenated flags + + diff --git a/core/src/test/resources/README.md b/core/src/test/resources/README.md new file mode 100644 index 000000000..70aa76cd6 --- /dev/null +++ b/core/src/test/resources/README.md @@ -0,0 +1,8 @@ +# Test resources + +## NAIP Virginia UTM overlaps + + 1. `m_3607717_sw_18_1_20160620_subset.tif` the southwest corner of NAIP m_3607717_sw_18_1. It is in its native CRS EPSG:26918 + 2. `m_3607824_se_17_1_20160620_subset.tif` the southeast corner of NAIP m_3607824_se_17_1. Overlaps number 1; It is to the east. It is in its native CRS EPSG:26917. + 3. `m_3607_box.tif` - an aribtrary burned in polygon in EPSG:4326 partially overlapping both of the above NAIP subsets 1 and 2. + diff --git a/core/src/test/resources/application.conf b/core/src/test/resources/application.conf new file mode 100644 index 000000000..b274441dd --- /dev/null +++ b/core/src/test/resources/application.conf @@ -0,0 +1,12 @@ +gdal { + settings { + options { + // See https://trac.osgeo.org/gdal/wiki/ConfigOptions for options + CPL_DEBUG = "OFF" + // TIFF_USE_OVR = "NO" + // GDAL_TIFF_INTERNAL_MASK = "YES" + } + // set this to `false` if CPL_DEBUG is `ON` + useExceptions: true + } +} \ No newline at end of file diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 378ae8e61..39e791fa3 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -30,15 +30,15 @@ log4j.logger.org.apache.spark.repl.Main=WARN log4j.logger.org.apache=ERROR log4j.logger.com.amazonaws=WARN -log4j.logger.geotrellis=INFO +log4j.logger.geotrellis=WARN # Settings to quiet third party logs that are too verbose log4j.logger.org.spark_project.jetty=WARN log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO -log4j.logger.astraea.spark.rasterframes=DEBUG -log4j.logger.astraea.spark.rasterframes.ref=TRACE +log4j.logger.org.locationtech.rasterframes=WARN +log4j.logger.org.locationtech.rasterframes.ref=WARN log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support diff --git a/core/src/test/resources/m_3607717_sw_18_1_20160620_subset.tif b/core/src/test/resources/m_3607717_sw_18_1_20160620_subset.tif new file mode 100644 index 000000000..862c23645 Binary files /dev/null and b/core/src/test/resources/m_3607717_sw_18_1_20160620_subset.tif differ diff --git a/core/src/test/resources/m_3607824_se_17_1_20160620_subset.tif b/core/src/test/resources/m_3607824_se_17_1_20160620_subset.tif new file mode 100644 index 000000000..2b2a7d497 Binary files /dev/null and b/core/src/test/resources/m_3607824_se_17_1_20160620_subset.tif differ diff --git a/core/src/test/resources/m_3607_box.tif b/core/src/test/resources/m_3607_box.tif new file mode 100644 index 000000000..50570db26 Binary files /dev/null and b/core/src/test/resources/m_3607_box.tif differ diff --git a/core/src/test/scala/Scratch.sc b/core/src/test/scala/Scratch.sc deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/test/scala/astraea/spark/rasterframes/GeometryOperationsSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/GeometryOperationsSpec.scala deleted file mode 100644 index 28d0bcb94..000000000 --- a/core/src/test/scala/astraea/spark/rasterframes/GeometryOperationsSpec.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes - -import java.nio.file.{Files, Paths} - -import geotrellis.proj4.LatLng -import geotrellis.vector.io._ -import geotrellis.vector.io.json.JsonFeatureCollection -import spray.json.DefaultJsonProtocol._ -import spray.json._ - -import scala.collection.JavaConversions._ - -/** - * - * - * @since 5/30/18 - */ -class GeometryOperationsSpec extends TestEnvironment with TestData { - val geoJson = { - val p = Paths.get(getClass.getResource("/L8-Labels-Elkton-VA.geojson").toURI) - Files.readAllLines(p).mkString("\n") - } - - describe("Geometery operations") { - import spark.implicits._ - it("should rasterize geometry") { - val rf = l8Sample(1).projectedRaster.toRF.withBounds() - - val features = geoJson.parseGeoJson[JsonFeatureCollection].getAllPolygonFeatures[JsObject]() - val df = features.map(f ⇒ ( - f.geom.reproject(LatLng, rf.crs).jtsGeom, - f.data.fields("id").asInstanceOf[JsNumber].value.intValue() - )).toDF("geom", "id") - - val toRasterize = rf.crossJoin(df) - - val tlm = rf.tileLayerMetadata.merge - - val (cols, rows) = tlm.layout.tileLayout.tileDimensions - - val rasterized = toRasterize.withColumn("rasterized", rasterize($"geom", $"bounds", $"id", cols, rows)) - - assert(rasterized.count() === df.count() * rf.count()) - assert(rasterized.select(tile_dimensions($"rasterized")).distinct().count() === 1) - val pixelCount = rasterized.select(agg_data_cells($"rasterized")).first() - assert(pixelCount < cols * rows) - - - toRasterize.createOrReplaceTempView("stuff") - val viaSQL = sql(s"select rf_rasterize(geom, bounds, id, $cols, $rows) as rasterized from stuff") - assert(viaSQL.select(agg_data_cells($"rasterized")).first === pixelCount) - - //rasterized.select($"rasterized".as[Tile]).foreach(t ⇒ t.renderPng(ColorMaps.IGBP).write("target/" + t.hashCode() + ".png")) - } - } -} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ReprojectGeometrySpec.scala b/core/src/test/scala/astraea/spark/rasterframes/ReprojectGeometrySpec.scala deleted file mode 100644 index 39ea3b1c1..000000000 --- a/core/src/test/scala/astraea/spark/rasterframes/ReprojectGeometrySpec.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2019 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes - -import com.vividsolutions.jts.geom._ -import geotrellis.proj4.{CRS, LatLng, Sinusoidal, WebMercator} -import org.apache.spark.sql.Encoders -import org.scalatest.{FunSpec, Matchers} - -/** - * Test for geometry reprojection. - * - * @since 11/29/18 - */ -class ReprojectGeometrySpec extends FunSpec - with TestEnvironment with Matchers { - import spark.implicits._ - - describe("Geometry reprojection") { - it("should allow reprojection geometry") { - // Note: Test data copied from ReprojectSpec in GeoTrellis - val fact = new GeometryFactory() - val latLng: Geometry = fact.createLineString(Array( - new Coordinate(-111.09374999999999, 34.784483415461345), - new Coordinate(-111.09374999999999, 43.29919735147067), - new Coordinate(-75.322265625, 43.29919735147067), - new Coordinate(-75.322265625, 34.784483415461345), - new Coordinate(-111.09374999999999, 34.784483415461345) - )) - - val webMercator: Geometry = fact.createLineString(Array( - new Coordinate(-12366899.680315234, 4134631.734001753), - new Coordinate(-12366899.680315234, 5357624.186564572), - new Coordinate(-8384836.254770693, 5357624.186564572), - new Coordinate(-8384836.254770693, 4134631.734001753), - new Coordinate(-12366899.680315234, 4134631.734001753) - )) - - withClue("both literal crs") { - - val df = Seq((latLng, webMercator)).toDF("ll", "wm") - - val rp = df.select( - reproject_geometry($"ll", LatLng, WebMercator) as "wm2", - reproject_geometry($"wm", WebMercator, LatLng) as "ll2", - reproject_geometry(reproject_geometry($"ll", LatLng, Sinusoidal), Sinusoidal, WebMercator) as "wm3" - ).as[(Geometry, Geometry, Geometry)] - - - val (wm2, ll2, wm3) = rp.first() - - wm2 should matchGeom(webMercator, 0.00001) - ll2 should matchGeom(latLng, 0.00001) - wm3 should matchGeom(webMercator, 0.00001) - } - - withClue("one literal crs") { - implicit val enc = Encoders.tuple(jtsGeometryEncoder, jtsGeometryEncoder, crsEncoder) - - val df = Seq((latLng, webMercator, LatLng: CRS)).toDF("ll", "wm", "llCRS") - - val rp = df.select( - reproject_geometry($"ll", $"llCRS", WebMercator) as "wm2", - reproject_geometry($"wm", WebMercator, $"llCRS") as "ll2", - reproject_geometry(reproject_geometry($"ll", $"llCRS", Sinusoidal), Sinusoidal, WebMercator) as "wm3" - ).as[(Geometry, Geometry, Geometry)] - - - val (wm2, ll2, wm3) = rp.first() - - wm2 should matchGeom(webMercator, 0.00001) - ll2 should matchGeom(latLng, 0.00001) - wm3 should matchGeom(webMercator, 0.00001) - - } - } - } - -} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ml/TileExploderSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/ml/TileExploderSpec.scala deleted file mode 100644 index 8883045e1..000000000 --- a/core/src/test/scala/astraea/spark/rasterframes/ml/TileExploderSpec.scala +++ /dev/null @@ -1,26 +0,0 @@ -package astraea.spark.rasterframes.ml - -import astraea.spark.rasterframes.{TestData, TestEnvironment} -import geotrellis.raster.Tile -import org.apache.spark.sql.functions.lit -/** - * - * @since 2/16/18 - */ -class TileExploderSpec extends TestEnvironment with TestData { - describe("Tile explode transformer") { - it("should explode tiles") { - import spark.implicits._ - val df = Seq[(Tile, Tile)]((byteArrayTile, byteArrayTile)).toDF("tile1", "tile2").withColumn("other", lit("stuff")) - - val exploder = new TileExploder() - val newSchema = exploder.transformSchema(df.schema) - - val exploded = exploder.transform(df) - assert(newSchema === exploded.schema) - assert(exploded.columns.length === 5) - assert(exploded.count() === 9) - write(exploded) - } - } -} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ref/RasterSourceSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/ref/RasterSourceSpec.scala deleted file mode 100644 index 1c1fb182a..000000000 --- a/core/src/test/scala/astraea/spark/rasterframes/ref/RasterSourceSpec.scala +++ /dev/null @@ -1,166 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.ref - -import java.net.URI - -import astraea.spark.rasterframes.TestEnvironment.ReadMonitor -import astraea.spark.rasterframes.ref.RasterSource.FileGeoTiffRasterSource -import astraea.spark.rasterframes.{TestData, TestEnvironment} -import geotrellis.raster.io.geotiff.GeoTiff -import geotrellis.vector.Extent -import org.apache.spark.sql.rf.RasterSourceUDT - -/** - * - * - * @since 8/22/18 - */ -class RasterSourceSpec extends TestEnvironment with TestData { - def sub(e: Extent) = { - val c = e.center - val w = e.width - val h = e.height - Extent(c.x, c.y, c.x + w * 0.1, c.y + h * 0.1) - } - - describe("General RasterSource") { - it("should identify as UDT") { - assert(new RasterSourceUDT() === new RasterSourceUDT()) - } - } - - describe("HTTP RasterSource") { - it("should support metadata querying over HTTP") { - withClue("remoteCOGSingleband") { - val src = RasterSource(remoteCOGSingleband1) - assert(!src.extent.isEmpty) - } - withClue("remoteCOGMultiband") { - val src = RasterSource(remoteCOGMultiband) - assert(!src.extent.isEmpty) - } - } - it("should read sub-tile") { - withClue("remoteCOGSingleband") { - val src = RasterSource(remoteCOGSingleband1) - val Left(raster) = src.read(sub(src.extent)) - assert(raster.size > 0 && raster.size < src.size) - } - withClue("remoteCOGMultiband") { - val src = RasterSource(remoteCOGMultiband) - //println("CoG size", src.size, src.dimensions) - val Right(raster) = src.read(sub(src.extent)) - //println("Subtile size", raster.size, raster.dimensions) - assert(raster.size > 0 && raster.size < src.size) - } - } - it("should Java serialize") { - import java.io._ - val src = RasterSource(remoteCOGSingleband1) - val buf = new java.io.ByteArrayOutputStream() - val out = new ObjectOutputStream(buf) - out.writeObject(src) - out.close() - - val data = buf.toByteArray - val in = new ObjectInputStream(new ByteArrayInputStream(data)) - val recovered = in.readObject().asInstanceOf[RasterSource] - assert(src.toString === recovered.toString) - } - } - describe("File RasterSource") { - it("should support metadata querying of file") { - val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toUri - val src = RasterSource(localSrc) - assert(!src.extent.isEmpty) - } - } - - describe("Caching") { - val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toUri - - trait Fixture { - val counter = ReadMonitor(false) - val src = RasterSource(localSrc, Some(counter)) - } - - it("should cache headers")(new Fixture { - val e = src.extent - assert(counter.reads === 1) - - val c = src.crs - val e2 = src.extent - val ct = src.cellType - assert(counter.reads === 1) - }) - - it("should Spark serialize caching")(new Fixture { - - import spark.implicits._ - - assert(src.isInstanceOf[FileGeoTiffRasterSource]) - - val e = src.extent - assert(counter.reads === 1) - - val df = Seq(src, src, src).toDS.repartition(3) - val src2 = df.collect()(1) - - val e2 = src2.extent - val ct = src2.cellType - - src2 match { - case fs: FileGeoTiffRasterSource ⇒ - fs.callback match { - case Some(cb: ReadMonitor) ⇒ assert(cb.reads === 1) - case o ⇒ fail(s"Expected '$o' to be a ReadMonitor") - } - case o ⇒ fail(s"Expected '$o' to be FileGeoTiffRasterSource") - } - }) - } - - describe("RasterSourceToTiles Expression") { - it("should read all tiles") { - val src = RasterSource(remoteMODIS) - - val subrasters = src.readAll().left.get - - val collected = subrasters.map(_.extent).reduceLeft(_.combine(_)) - - assert(src.extent.xmin === collected.xmin +- 0.01) - assert(src.extent.ymin === collected.ymin +- 0.01) - assert(src.extent.xmax === collected.xmax +- 0.01) - assert(src.extent.ymax === collected.ymax +- 0.01) - - val totalCells = subrasters.map(_.size).sum - - assert(totalCells === src.size) - - subrasters.zipWithIndex.foreach{case (r, i) ⇒ - // TODO: how to test? - GeoTiff(r, src.crs).write(s"target/$i.tiff") - } - } - } -} diff --git a/core/src/test/scala/examples/Classification.scala b/core/src/test/scala/examples/Classification.scala deleted file mode 100644 index 4aebcf742..000000000 --- a/core/src/test/scala/examples/Classification.scala +++ /dev/null @@ -1,160 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.ml.{NoDataFilter, TileExploder} -import geotrellis.raster._ -import geotrellis.raster.io.geotiff.reader.GeoTiffReader -import geotrellis.raster.render.{ColorRamps, IndexedColorMap} -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.classification.DecisionTreeClassifier -import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator -import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.ml.tuning.{CrossValidator, ParamGridBuilder} -import org.apache.spark.sql._ - -object Classification extends App { - -// // Utility for reading imagery from our test data set - def readTiff(name: String) = GeoTiffReader.readSingleband(getClass.getResource(s"/$name").getPath) - - implicit val spark = SparkSession.builder() - .master("local[*]") - .appName(getClass.getName) - .getOrCreate() - .withRasterFrames - - import spark.implicits._ - - // The first step is to load multiple bands of imagery and construct - // a single RasterFrame from them. - val filenamePattern = "L8-%s-Elkton-VA.tiff" - val bandNumbers = 2 to 7 - val bandColNames = bandNumbers.map(b ⇒ s"band_$b").toArray - val tileSize = 10 - - // For each identified band, load the associated image file - val joinedRF = bandNumbers - .map { b ⇒ (b, filenamePattern.format("B" + b)) } - .map { case (b, f) ⇒ (b, readTiff(f)) } - .map { case (b, t) ⇒ t.projectedRaster.toRF(tileSize, tileSize, s"band_$b") } - .reduce(_ spatialJoin _) - - // We should see a single spatial_key column along with 4 columns of tiles. - joinedRF.printSchema() - - // Similarly pull in the target label data. - val targetCol = "target" - - // Load the target label raster. We have to convert the cell type to - // Double to meet expectations of SparkML - val target = readTiff(filenamePattern.format("Labels")) - .mapTile(_.convert(DoubleConstantNoDataCellType)) - .projectedRaster - .toRF(tileSize, tileSize, targetCol) - - // Take a peek at what kind of label data we have to work with. - target.select(agg_stats(target(targetCol))).show - - val abt = joinedRF.spatialJoin(target) - - // SparkML requires that each observation be in its own row, and those - // observations be packed into a single `Vector`. The first step is to - // "explode" the tiles into a single row per cell/pixel - val exploder = new TileExploder() - - val noDataFilter = new NoDataFilter() - .setInputCols(bandColNames :+ targetCol) - - // To "vectorize" the the band columns we use the SparkML `VectorAssembler` - val assembler = new VectorAssembler() - .setInputCols(bandColNames) - .setOutputCol("features") - - // Using a decision tree for classification - val classifier = new DecisionTreeClassifier() - .setLabelCol(targetCol) - .setFeaturesCol(assembler.getOutputCol) - - // Assemble the model pipeline - val pipeline = new Pipeline() - .setStages(Array(exploder, noDataFilter, assembler, classifier)) - - // Configure how we're going to evaluate our model's performance. - val evaluator = new MulticlassClassificationEvaluator() - .setLabelCol(targetCol) - .setPredictionCol("prediction") - .setMetricName("f1") - - // Use a parameter grid to determine what the optimal max tree depth is for this data - val paramGrid = new ParamGridBuilder() - //.addGrid(classifier.maxDepth, Array(1, 2, 3, 4)) - .build() - - // Configure the cross validator - val trainer = new CrossValidator() - .setEstimator(pipeline) - .setEvaluator(evaluator) - .setEstimatorParamMaps(paramGrid) - .setNumFolds(4) - - // Push the "go" button - val model = trainer.fit(abt) - - // Format the `paramGrid` settings resultant model - val metrics = model.getEstimatorParamMaps - .map(_.toSeq.map(p ⇒ s"${p.param.name} = ${p.value}")) - .map(_.mkString(", ")) - .zip(model.avgMetrics) - - // Render the parameter/performance association - metrics.toSeq.toDF("params", "metric").show(false) - - // Score the original data set, including cells - // without target values. - val scored = model.bestModel.transform(joinedRF) - - // Add up class membership results - scored.groupBy($"prediction" as "class").count().show - - scored.show(10) - - val tlm = joinedRF.tileLayerMetadata.left.get - - val retiled = scored.groupBy($"spatial_key").agg( - assemble_tile( - $"column_index", $"row_index", $"prediction", - tlm.tileCols, tlm.tileRows, IntConstantNoDataCellType - ) - ) - - val rf = retiled.asRF($"spatial_key", tlm) - - val raster = rf.toRaster($"prediction", 186, 169) - - val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 until 3).toArray) - ) - - raster.tile.renderPng(clusterColors).write("target/scala-2.11/tut/ml/classified.png") - - spark.stop() -} diff --git a/core/src/test/scala/examples/Clustering.scala b/core/src/test/scala/examples/Clustering.scala deleted file mode 100644 index 2f8d4ce1f..000000000 --- a/core/src/test/scala/examples/Clustering.scala +++ /dev/null @@ -1,108 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.ml.TileExploder -import geotrellis.raster.ByteConstantNoDataCellType -import geotrellis.raster.io.geotiff.reader.GeoTiffReader -import geotrellis.raster.render.{ColorRamps, IndexedColorMap} -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.clustering.{KMeans, KMeansModel} -import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.sql._ - -object Clustering extends App { - - // Utility for reading imagery from our test data set - def readTiff(name: String) = GeoTiffReader.readSingleband(getClass.getResource(s"/$name").getPath) - - implicit val spark = SparkSession.builder().master("local[*]").appName(getClass.getName).getOrCreate().withRasterFrames - - import spark.implicits._ - - // The first step is to load multiple bands of imagery and construct - // a single RasterFrame from them. - val filenamePattern = "L8-B%d-Elkton-VA.tiff" - val bandNumbers = 1 to 7 - val bandColNames = bandNumbers.map(b ⇒ s"band_$b").toArray - - // For each identified band, load the associated image file - val joinedRF = bandNumbers - .map { b ⇒ (b, filenamePattern.format(b)) } - .map { case (b,f) ⇒ (b, readTiff(f)) } - .map { case (b, t) ⇒ t.projectedRaster.toRF(s"band_$b") } - .reduce(_ spatialJoin _) - - // We should see a single spatial_key column along with 4 columns of tiles. - joinedRF.printSchema() - - // SparkML requires that each observation be in its own row, and those - // observations be packed into a single `Vector`. The first step is to - // "explode" the tiles into a single row per cell/pixel - val exploder = new TileExploder() - - // To "vectorize" the the band columns we use the SparkML `VectorAssembler` - val assembler = new VectorAssembler() - .setInputCols(bandColNames) - .setOutputCol("features") - - // Configure our clustering algorithm - val k = 5 - val kmeans = new KMeans().setK(k) - - // Combine the two stages - val pipeline = new Pipeline().setStages(Array(exploder, assembler, kmeans)) - - // Compute clusters - val model = pipeline.fit(joinedRF) - - // Run the data through the model to assign cluster IDs to each - val clustered = model.transform(joinedRF) - clustered.show(8) - - // If we want to inspect the model statistics, the SparkML API requires us to go - // through this unfortunate contortion: - val clusterResults = model.stages.collect{ case km: KMeansModel ⇒ km}.head - - // Compute sum of squared distances of points to their nearest center - val metric = clusterResults.computeCost(clustered) - println("Within set sum of squared errors: " + metric) - - val tlm = joinedRF.tileLayerMetadata.left.get - - val retiled = clustered.groupBy($"spatial_key").agg( - assemble_tile( - $"column_index", $"row_index", $"prediction", - tlm.tileCols, tlm.tileRows, ByteConstantNoDataCellType) - ) - - val rf = retiled.asRF($"spatial_key", tlm) - - val raster = rf.toRaster($"prediction", 186, 169) - - val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 until k).toArray) - ) - - raster.tile.renderPng(clusterColors).write("clustered.png") - - spark.stop() -} diff --git a/core/src/test/scala/examples/CreatingRasterFrames.scala b/core/src/test/scala/examples/CreatingRasterFrames.scala index f7a69043a..8b5c00c72 100644 --- a/core/src/test/scala/examples/CreatingRasterFrames.scala +++ b/core/src/test/scala/examples/CreatingRasterFrames.scala @@ -27,19 +27,17 @@ package examples object CreatingRasterFrames extends App { // # Creating RasterFrames // -// There are a number of ways to create a `RasterFrame`, as enumerated in the sections below. +// There are a number of ways to create a `RasterFrameLayer`, as enumerated in the sections below. // // ## Initialization // // First, some standard `import`s: - import astraea.spark.rasterframes._ + import org.locationtech.rasterframes._ import geotrellis.raster._ - import geotrellis.raster.render._ - import geotrellis.spark.io._ import geotrellis.raster.io.geotiff.SinglebandGeoTiff + import geotrellis.spark.io._ import org.apache.spark.sql._ - import org.apache.spark.sql.functions._ // Next, initialize the `SparkSession`, and call the `withRasterFrames` method on it: @@ -47,27 +45,25 @@ object CreatingRasterFrames extends App { master("local[*]").appName("RasterFrames"). getOrCreate(). withRasterFrames - - import spark.implicits._ spark.sparkContext.setLogLevel("ERROR") // ## From `ProjectedExtent` // -// The simplest mechanism for getting a RasterFrame is to use the `toRF(tileCols, tileRows)` extension method on `ProjectedRaster`. +// The simplest mechanism for getting a RasterFrameLayer is to use the `toLayer(tileCols, tileRows)` extension method on `ProjectedRaster`. val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - val rf = scene.projectedRaster.toRF(128, 128) + val rf = scene.projectedRaster.toLayer(128, 128) rf.show(5, false) // ## From `TileLayerRDD` // -// Another option is to use a GeoTrellis [`LayerReader`](https://docs.geotrellis.io/en/latest/guide/tile-backends.html), to get a `TileLayerRDD` for which there's also a `toRF` extension method. +// Another option is to use a GeoTrellis [`LayerReader`](https://docs.geotrellis.io/en/latest/guide/tile-backends.html), to get a `TileLayerRDD` for which there's also a `toLayer` extension method. // ## Inspecting Structure // -// `RasterFrame` has a number of methods providing access to metadata about the contents of the RasterFrame. +// `RasterFrameLayer` has a number of methods providing access to metadata about the contents of the RasterFrameLayer. // // ### Tile Column Names diff --git a/core/src/test/scala/examples/Exporting.scala b/core/src/test/scala/examples/Exporting.scala index 247e93944..25fa321c1 100644 --- a/core/src/test/scala/examples/Exporting.scala +++ b/core/src/test/scala/examples/Exporting.scala @@ -20,13 +20,11 @@ package examples import java.nio.file.Files -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster._ +import geotrellis.raster.io.geotiff.SinglebandGeoTiff import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.{GeoTiff, SinglebandGeoTiff} import geotrellis.spark.{LayerId, SpatialKey} -import geotrellis.spark.io.LayerWriter -import geotrellis.spark.io.file.{FileAttributeStore, FileLayerWriter} import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import spray.json.JsValue @@ -40,7 +38,7 @@ object Exporting extends App { import spark.implicits._ val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - val rf = scene.projectedRaster.toRF(128, 128).cache() + val rf = scene.projectedRaster.toLayer(128, 128).cache() // While the goal of RasterFrames is to make it as easy as possible to do your geospatial analysis with a single // construct, it is helpful to be able to transform it into other representations for various use cases. @@ -54,17 +52,17 @@ object Exporting extends App { // The @scaladoc[`tile_to_array`][tile_to_array] column function requires a type parameter to indicate the array element // type you would like used. The following types may be used: `Int`, `Double`, `Byte`, `Short`, `Float` - val withArrays = rf.withColumn("tileData", tile_to_array_int($"tile")).drop("tile") + val withArrays = rf.withColumn("tileData", rf_tile_to_array_int($"tile")).drop("tile") withArrays.show(5, 40) // You can convert the data back to an array, but you have to specify the target tile dimensions. - val tileBack = withArrays.withColumn("tileAgain", array_to_tile($"tileData", 128, 128)) + val tileBack = withArrays.withColumn("tileAgain", rf_array_to_tile($"tileData", 128, 128)) tileBack.drop("tileData").show(5, 40) // Note that the created tile will not have a `NoData` value associated with it. Here's how you can do that: - val tileBackAgain = withArrays.withColumn("tileAgain", with_no_data(array_to_tile($"tileData", 128, 128), 3)) + val tileBackAgain = withArrays.withColumn("tileAgain", rf_with_no_data(rf_array_to_tile($"tileData", 128, 128), 3)) tileBackAgain.drop("tileData").show(5, 50) // ## Writing to Parquet @@ -75,15 +73,13 @@ object Exporting extends App { // the imagery types. // // - // Let's assume we have a RasterFrame we've done some fancy processing on: - - import geotrellis.raster.equalization._ + // Let's assume we have a RasterFrameLayer we've done some fancy processing on: val equalizer = udf((t: Tile) => t.equalize()) - val equalized = rf.withColumn("equalized", equalizer($"tile")).asRF + val equalized = rf.withColumn("equalized", equalizer($"tile")).asLayer equalized.printSchema - equalized.select(agg_stats($"tile")).show(false) - equalized.select(agg_stats($"equalized")).show(false) + equalized.select(rf_agg_stats($"tile")).show(false) + equalized.select(rf_agg_stats($"equalized")).show(false) // We write it out just like any other DataFrame, including the ability to specify partitioning: @@ -102,12 +98,12 @@ object Exporting extends App { val rf2 = spark.read.parquet(filePath) rf2.printSchema - equalized.select(agg_stats($"tile")).show(false) - equalized.select(agg_stats($"equalized")).show(false) + equalized.select(rf_agg_stats($"tile")).show(false) + equalized.select(rf_agg_stats($"equalized")).show(false) // ## Converting to `RDD` and `TileLayerRDD` // - // Since a `RasterFrame` is just a `DataFrame` with extra metadata, the method + // Since a `RasterFrameLayer` is just a `DataFrame` with extra metadata, the method // @scaladoc[`DataFrame.rdd`][rdd] is available for simple conversion back to `RDD` space. The type returned // by `.rdd` is dependent upon how you select it. @@ -122,14 +118,14 @@ object Exporting extends App { showType(rf.select(rf.spatialKeyColumn, $"tile").as[(SpatialKey, Tile)].rdd) // If your goal convert a single tile column with its spatial key back to a `TileLayerRDD[K]`, then there's an additional - // extension method on `RasterFrame` called [`toTileLayerRDD`][toTileLayerRDD], which preserves the tile layer metadata, + // extension method on `RasterFrameLayer` called [`toTileLayerRDD`][toTileLayerRDD], which preserves the tile layer metadata, // enhancing interoperation with GeoTrellis RDD extension methods. showType(rf.toTileLayerRDD($"tile".as[Tile])) // ## Exporting a Raster // - // For the purposes of debugging, the RasterFrame tiles can be reassembled back into a raster for viewing. However, + // For the purposes of debugging, the RasterFrameLayer tiles can be reassembled back into a raster for viewing. However, // keep in mind that this will download all the data to the driver, and reassemble it in-memory. So it's not appropriate // for very large coverages. // @@ -151,7 +147,7 @@ object Exporting extends App { // [*Download GeoTIFF*](rf-raster.tiff) // # Exporting to a GeoTrellis Layer - // First, convert the RasterFrame into a TileLayerRDD. The return type is an Either; + // First, convert the RasterFrameLayer into a TileLayerRDD. The return type is an Either; // the `left` side is for spatial-only keyed data val tlRDD = equalized.toTileLayerRDD($"equalized").left.get diff --git a/core/src/test/scala/examples/LocalArithmetic.scala b/core/src/test/scala/examples/LocalArithmetic.scala index ddf666e96..428fcc64a 100644 --- a/core/src/test/scala/examples/LocalArithmetic.scala +++ b/core/src/test/scala/examples/LocalArithmetic.scala @@ -19,7 +19,7 @@ package examples -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster.io.geotiff.SinglebandGeoTiff import geotrellis.spark.io.kryo.KryoRegistrator import org.apache.spark.serializer.KryoSerializer @@ -49,15 +49,15 @@ object LocalArithmetic extends App { val joinedRF = bandNumbers. map { b ⇒ (b, filenamePattern.format(b)) }. map { case (b, f) ⇒ (b, readTiff(f)) }. - map { case (b, t) ⇒ t.projectedRaster.toRF(s"band_$b") }. + map { case (b, t) ⇒ t.projectedRaster.toLayer(s"band_$b") }. reduce(_ spatialJoin _) - val addRF = joinedRF.withColumn("1+2", local_add(joinedRF("band_1"), joinedRF("band_2"))).asRF - val divideRF = joinedRF.withColumn("1/2", local_divide(joinedRF("band_1"), joinedRF("band_2"))).asRF + val addRF = joinedRF.withColumn("1+2", rf_local_add(joinedRF("band_1"), joinedRF("band_2"))).asLayer + val divideRF = joinedRF.withColumn("1/2", rf_local_divide(joinedRF("band_1"), joinedRF("band_2"))).asLayer addRF.select("1+2").collect().apply(0) .getClass - val raster = divideRF.select(tile_sum(divideRF("1/2")), - tile_sum(joinedRF("band_1")), tile_sum(joinedRF("band_2"))) + val raster = divideRF.select(rf_tile_sum(divideRF("1/2")), + rf_tile_sum(joinedRF("band_1")), rf_tile_sum(joinedRF("band_2"))) raster.show(1) } \ No newline at end of file diff --git a/core/src/test/scala/examples/Masking.scala b/core/src/test/scala/examples/Masking.scala index bc9c59213..6270bcef1 100644 --- a/core/src/test/scala/examples/Masking.scala +++ b/core/src/test/scala/examples/Masking.scala @@ -1,12 +1,11 @@ package examples -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.sql._ -import geotrellis.raster.{mask => _, _} import geotrellis.raster.render._ +import geotrellis.raster.{mask => _, _} +import org.apache.spark.sql._ import org.apache.spark.sql.functions._ -import astraea.spark.rasterframes.stats.{CellHistogram=>CH} object Masking extends App { @@ -25,18 +24,18 @@ object Masking extends App { val joinedRF = bandNumbers. map { b ⇒ (b, filenamePattern.format(b)) }. map { case (b, f) ⇒ (b, readTiff(f)) }. - map { case (b, t) ⇒ t.projectedRaster.toRF(s"band_$b") }. + map { case (b, t) ⇒ t.projectedRaster.toLayer(s"band_$b") }. reduce(_ spatialJoin _) val threshold = udf((t: Tile) => { t.convert(IntConstantNoDataCellType).map(x => if (x > 10500) x else NODATA) } ) - val withMaskedTile = joinedRF.withColumn("maskTile", threshold(joinedRF("band_1"))).asRF + val withMaskedTile = joinedRF.withColumn("maskTile", threshold(joinedRF("band_1"))).asLayer - withMaskedTile.select(no_data_cells(withMaskedTile("maskTile"))).show() + withMaskedTile.select(rf_no_data_cells(withMaskedTile("maskTile"))).show() - val masked = withMaskedTile.withColumn("masked", mask(joinedRF("band_2"), joinedRF("maskTile"))).asRF + val masked = withMaskedTile.withColumn("masked", rf_mask(joinedRF("band_2"), joinedRF("maskTile"))).asLayer val maskRF = masked.toRaster(masked("masked"), 466, 428) val b2 = masked.toRaster(masked("band_2"), 466, 428) diff --git a/core/src/test/scala/examples/MeanValue.scala b/core/src/test/scala/examples/MeanValue.scala index d2190a241..2ee264469 100644 --- a/core/src/test/scala/examples/MeanValue.scala +++ b/core/src/test/scala/examples/MeanValue.scala @@ -19,10 +19,9 @@ package examples -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster.io.geotiff.SinglebandGeoTiff import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.functions._ /** * Compute the cell mean value of an image. @@ -40,12 +39,12 @@ object MeanValue extends App { val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - val rf = scene.projectedRaster.toRF(128, 128) // <-- tile size + val rf = scene.projectedRaster.toLayer(128, 128) // <-- tile size rf.printSchema val tileCol = rf("tile") - rf.agg(agg_no_data_cells(tileCol), agg_data_cells(tileCol), agg_mean(tileCol)).show(false) + rf.agg(rf_agg_no_data_cells(tileCol), rf_agg_data_cells(tileCol), rf_agg_mean(tileCol)).show(false) spark.stop() } diff --git a/core/src/test/scala/examples/NDVI.scala b/core/src/test/scala/examples/NDVI.scala index 971dfd8d4..48a6f6e51 100644 --- a/core/src/test/scala/examples/NDVI.scala +++ b/core/src/test/scala/examples/NDVI.scala @@ -20,7 +20,7 @@ package examples import java.nio.file.{Files, Paths} -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ import geotrellis.raster._ import geotrellis.raster.render._ import geotrellis.raster.io.geotiff.{GeoTiff, SinglebandGeoTiff} @@ -46,8 +46,8 @@ object NDVI extends App { import spark.implicits._ - def redBand = readTiff("L8-B4-Elkton-VA.tiff").projectedRaster.toRF("red_band") - def nirBand = readTiff("L8-B5-Elkton-VA.tiff").projectedRaster.toRF("nir_band") + def redBand = readTiff("L8-B4-Elkton-VA.tiff").projectedRaster.toLayer("red_band") + def nirBand = readTiff("L8-B5-Elkton-VA.tiff").projectedRaster.toLayer("nir_band") val ndvi = udf((red: Tile, nir: Tile) => { val redd = red.convert(DoubleConstantNoDataCellType) @@ -55,7 +55,7 @@ object NDVI extends App { (nird - redd) / (nird + redd) }) - val rf = redBand.spatialJoin(nirBand).withColumn("ndvi", ndvi($"red_band", $"nir_band")).asRF + val rf = redBand.spatialJoin(nirBand).withColumn("ndvi", ndvi($"red_band", $"nir_band")).asLayer rf.printSchema() diff --git a/core/src/test/scala/examples/NaturalColorComposite.scala b/core/src/test/scala/examples/NaturalColorComposite.scala index 3dee4092c..1a3e212ac 100644 --- a/core/src/test/scala/examples/NaturalColorComposite.scala +++ b/core/src/test/scala/examples/NaturalColorComposite.scala @@ -20,11 +20,8 @@ package examples -import java.nio.file.{CopyOption, StandardCopyOption} - +import geotrellis.raster.io.geotiff.SinglebandGeoTiff import geotrellis.raster.{MultibandTile, UByteConstantNoDataCellType} -import geotrellis.raster.io.geotiff.{GeoTiff, SinglebandGeoTiff} -import geotrellis.raster.render._ /** * diff --git a/core/src/test/scala/examples/Scratch.scala b/core/src/test/scala/examples/Scratch.scala deleted file mode 100644 index fca8c4785..000000000 --- a/core/src/test/scala/examples/Scratch.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import astraea.spark.rasterframes._ -import geotrellis.spark.io.kryo.KryoRegistrator -import org.apache.spark.serializer.KryoSerializer -import org.apache.spark.sql._ - -/** - * Boilerplate test run file - * - * @since 10/8/17 - */ -object Scratch extends App { - implicit val spark = SparkSession.builder() - .master("local[*]") - .appName(getClass.getName) - .config("spark.serializer", classOf[KryoSerializer].getName) - .config("spark.kryoserializer.buffer.max", "500m") - .config("spark.kryo.registrationRequired", "false") - .config("spark.kryo.registrator", classOf[KryoRegistrator].getName) - .getOrCreate() - .withRasterFrames - - import spark.implicits._ - - // Your Spark code here..... - -} diff --git a/core/src/test/scala/examples/Tour.scala b/core/src/test/scala/examples/Tour.scala deleted file mode 100644 index d69cb5a1c..000000000 --- a/core/src/test/scala/examples/Tour.scala +++ /dev/null @@ -1,140 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ -import geotrellis.raster.io.geotiff._ -import geotrellis.raster.{ByteConstantNoDataCellType, Tile} -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.ml.TileExploder -import geotrellis.raster.render.{ColorRamps, IndexedColorMap} -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.clustering.KMeans -import org.apache.spark.ml.feature.VectorAssembler - -/** - * Example tour of some general features in RasterFrames - * - * @since 10/24/17 - */ -object Tour extends App { - implicit val spark = SparkSession.builder() - .master("local[*]") - .appName(getClass.getName) - .getOrCreate() - .withRasterFrames - - import spark.implicits._ - - // Read in a geo-referenced image - val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - - // Convert it to a raster frame, discretizing it into the given tile size. - val rf = scene.projectedRaster.toRF(64, 64) - - // See how many tiles we have after discretization - println("Tile count: " + rf.count()) - - // Take a peek at what we're working with - rf.show(8, false) - - // Confirm we have equally sized tiles - rf.select(tile_dimensions($"tile")).distinct().show() - - // Count the number of no-data cells - rf.select(agg_no_data_cells($"tile")).show(false) - - // Compute per-tile statistics - rf.select(tile_stats($"tile")).show(8, false) - - // Compute some aggregate stats over all cells - rf.select(agg_stats($"tile")).show(false) - - // Create a Spark UDT to perform contrast adjustment via GeoTrellis - val contrast = udf((t: Tile) ⇒ t.sigmoidal(0.2, 10)) - - // Let's contrast adjust the tile column - val withAdjusted = rf.withColumn("adjusted", contrast($"tile")).asRF - - // Show the stats for the adjusted version - withAdjusted.select(agg_stats($"adjusted")).show(false) - - // Reassemble into a raster and save to a file - val raster = withAdjusted.toRaster($"adjusted", 774, 500) - GeoTiff(raster).write("contrast-adjusted.tiff") - - // Perform some arbitrary local ops between columns and render - val withOp = withAdjusted.withColumn("op", local_subtract($"tile", $"adjusted")).asRF - val raster2 = withOp.toRaster($"op", 774, 500) - GeoTiff(raster2).write("with-op.tiff") - - - // Perform k-means clustering - val k = 4 - - // SparkML doesn't like NoData/NaN values, so we set the no-data value to something less offensive - val forML = rf.select(rf.spatialKeyColumn, with_no_data($"tile", 99999) as "tile").asRF - - // First we instantiate the transformer that converts tile rows into cell rows. - val exploder = new TileExploder() - - // This transformer wraps the pixel values in a vector. - // Could use this with multiple bands - val assembler = new VectorAssembler(). - setInputCols(Array("tile")). - setOutputCol("features") - - // Or clustering algorithm - val kmeans = new KMeans().setK(k) - - // Construct the ML pipeline - val pipeline = new Pipeline().setStages(Array(exploder, assembler, kmeans)) - - // Compute the model - val model = pipeline.fit(forML) - - // Score the data - val clusteredCells = model.transform(forML) - - clusteredCells.show() - - clusteredCells.groupBy("prediction").count().show - - // Reassembling the clustering results takes a number of steps. - val tlm = rf.tileLayerMetadata.left.get - - // RasterFrames provides a special aggregation function for assembling tiles from cells with column/row indexes - val retiled = clusteredCells.groupBy(forML.spatialKeyColumn).agg( - assemble_tile($"column_index", $"row_index", $"prediction", tlm.tileCols, tlm.tileRows, ByteConstantNoDataCellType) - ) - - val clusteredRF = retiled.asRF($"spatial_key", tlm) - - val raster3 = clusteredRF.toRaster($"prediction", 774, 500) - - val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 until k).toArray) - ) - - GeoTiff(raster3).copy(options = GeoTiffOptions(clusterColors)).write("clustered.tiff") - - spark.stop() -} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ExplodeSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala similarity index 62% rename from core/src/test/scala/astraea/spark/rasterframes/ExplodeSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala index a06b6444b..da0af2397 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/ExplodeSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes import geotrellis.raster._ import geotrellis.raster.resample.NearestNeighbor @@ -43,14 +45,14 @@ class ExplodeSpec extends TestEnvironment with TestData { write(query) assert(query.select("cell_0", "cell_1").as[(Double, Double)].collect().forall(_ == ((1.0, 2.0)))) val query2 = sql( - """|select rf_tile_dimensions(tiles) as dims, rf_explode_tiles(tiles) from ( + """|select rf_dimensions(tiles) as dims, rf_explode_tiles(tiles) from ( |select rf_make_constant_tile(1, 10, 10, 'int8raw') as tiles) |""".stripMargin) write(query2) assert(query2.columns.length === 4) val df = Seq[(Tile, Tile)]((byteArrayTile, byteArrayTile)).toDF("tile1", "tile2") - val exploded = df.select(explode_tiles($"tile1", $"tile2")) + val exploded = df.select(rf_explode_tiles($"tile1", $"tile2")) //exploded.printSchema() assert(exploded.columns.length === 4) assert(exploded.count() === 9) @@ -59,17 +61,17 @@ class ExplodeSpec extends TestEnvironment with TestData { it("should explode tiles with random sampling") { val df = Seq[(Tile, Tile)]((byteArrayTile, byteArrayTile)).toDF("tile1", "tile2") - val exploded = df.select(explode_tiles_sample(0.5, $"tile1", $"tile2")) + val exploded = df.select(rf_explode_tiles_sample(0.5, $"tile1", $"tile2")) assert(exploded.columns.length === 4) assert(exploded.count() < 9) } it("should handle null tiles") { val df = Seq[Tile](null, byteArrayTile, null, byteArrayTile, null).toDF("tile1") - val exploded = df.select(explode_tiles($"tile1")) + val exploded = df.select(rf_explode_tiles($"tile1")) assert(exploded.count === byteArrayTile.size * 2) val df2 = Seq[(Tile, Tile)]((byteArrayTile, null), (null, byteArrayTile), (byteArrayTile, byteArrayTile)).toDF("tile1", "tile2") - val exploded2 = df2.select(explode_tiles($"tile1", $"tile2")) + val exploded2 = df2.select(rf_explode_tiles($"tile1", $"tile2")) assert(exploded2.count === byteArrayTile.size * 3) } @@ -77,7 +79,7 @@ class ExplodeSpec extends TestEnvironment with TestData { // Create a tile with a single (wierd) no-data value val tile: Tile = UShortArrayTile(rangeArray(9, _.toShort), 3, 3, 5.toShort) val cells = Seq(tile).toDF("tile") - .select(explode_tiles($"tile")) + .select(rf_explode_tiles($"tile")) .select($"tile".as[Double]) .collect() @@ -87,7 +89,7 @@ class ExplodeSpec extends TestEnvironment with TestData { it("should handle user-defined NoData values in tile sampler") { val tiles = allTileTypes.filter(t ⇒ !t.isInstanceOf[BitArrayTile]).map(_.withNoData(Some(3))) val cells = tiles.toDF("tile") - .select(explode_tiles($"tile")) + .select(rf_explode_tiles($"tile")) .select($"tile".as[Double]) .collect() cells.count(_.isNaN) should be(tiles.size) @@ -103,53 +105,92 @@ class ExplodeSpec extends TestEnvironment with TestData { val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType) val df = Seq[Tile](tile).toDF("tile") - val arrayDF = df.select(tile_to_array_double($"tile").as[Array[Double]]) + val arrayDF = df.select(rf_tile_to_array_double($"tile").as[Array[Double]]) arrayDF.first().sum should be (110.0 +- 0.0001) } it("should convert an array into a tile") { - val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType) + val tile = TestData.randomTile(10, 10, FloatCellType) val df = Seq[Tile](tile, null).toDF("tile") - val arrayDF = df.withColumn("tileArray", tile_to_array_double($"tile")) + val arrayDF = df.withColumn("tileArray", rf_tile_to_array_double($"tile")) - val back = arrayDF.withColumn("backToTile", array_to_tile($"tileArray", 10, 10)) + val back = arrayDF.withColumn("backToTile", rf_array_to_tile($"tileArray", 10, 10)) val result = back.select($"backToTile".as[Tile]).first assert(result.toArrayDouble() === tile.toArrayDouble()) - val hasNoData = back.withColumn("with_no_data", with_no_data($"backToTile", 0)) + // Same round trip, but with SQL expression for rf_array_to_tile + val resultSql = arrayDF.selectExpr("rf_array_to_tile(tileArray, 10, 10) as backToTile").as[Tile].first + + assert(resultSql.toArrayDouble() === tile.toArrayDouble()) - val result2 = hasNoData.select($"with_no_data".as[Tile]).first + val hasNoData = back.withColumn("withNoData", rf_with_no_data($"backToTile", 0)) + + val result2 = hasNoData.select($"withNoData".as[Tile]).first assert(result2.cellType.asInstanceOf[UserDefinedNoData[_]].noDataValue === 0) } it("should reassemble single exploded tile") { - val df = Seq[Tile](byteArrayTile).toDF("tile") - .select(explode_tiles($"tile")) + val tile = TestData.randomTile(10, 10, FloatCellType) + val df = Seq[Tile](tile).toDF("tile") + .select(rf_explode_tiles($"tile")) - val assembled = df.agg(assemble_tile( + val assembled = df.agg( + rf_assemble_tile( COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, TILE_COLUMN, - 3, 3, byteArrayTile.cellType + 10, 10, tile.cellType )).as[Tile] val result = assembled.first() - assert(result === byteArrayTile) + assert(result === tile) + + val assembledSqlExpr = df.selectExpr("rf_assemble_tile(column_index, row_index, tile, 10, 10)") + + val resultSql = assembledSqlExpr.as[Tile].first() + assert(resultSql === tile) + + checkDocs("rf_assemble_tile") + } + + it("should reassemble single exploded tile with user-defined nodata") { + val ct = FloatUserDefinedNoDataCellType(-99) + val tile = TestData.injectND(3)(TestData.randomTile(5, 5, ct)) + val df = Seq[Tile](tile).toDF("tile") + .select(rf_explode_tiles($"tile")) + + val assembled = df.agg(rf_assemble_tile( + COLUMN_INDEX_COLUMN, + ROW_INDEX_COLUMN, + TILE_COLUMN, + 5, 5, ct + )).as[Tile] + + val result = assembled.first() + assert(result === tile) + + // and with SQL API + logger.info(df.schema.treeString) + + val assembledSqlExpr = df.selectExpr(s"rf_convert_cell_type(rf_assemble_tile(column_index, row_index, tile, 5, 5), '${ct.toString()}') as tile") + + val resultSql = assembledSqlExpr.as[Tile].first() + assert(resultSql === tile) + assert(resultSql.cellType === ct) } it("should reassemble multiple exploded tiles") { val image = sampleSmallGeoTiff - val tinyTiles = image.projectedRaster.toRF(10, 10) + val tinyTiles = image.projectedRaster.toLayer(10, 10) - val exploded = tinyTiles.select(tinyTiles.spatialKeyColumn, explode_tiles(tinyTiles.tileColumns.head)) - - //exploded.printSchema() + val exploded = tinyTiles.select(tinyTiles.spatialKeyColumn, rf_explode_tiles(tinyTiles.tileColumns.head)) val assembled = exploded.groupBy(tinyTiles.spatialKeyColumn) - .agg(assemble_tile( + .agg( + rf_assemble_tile( COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, TILE_COLUMN, @@ -158,7 +199,7 @@ class ExplodeSpec extends TestEnvironment with TestData { val tlm = tinyTiles.tileLayerMetadata.left.get - val rf = assembled.asRF(SPATIAL_KEY_COLUMN, tlm) + val rf = assembled.asLayer(SPATIAL_KEY_COLUMN, tlm) val (cols, rows) = image.tile.dimensions diff --git a/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala similarity index 64% rename from core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala index 81c2d9202..eeea68544 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,16 +15,18 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes -import astraea.spark.rasterframes.util.SubdivideSupport._ +package org.locationtech.rasterframes + import geotrellis.proj4.LatLng import geotrellis.raster.{ByteCellType, GridBounds, TileLayout} +import geotrellis.spark.tiling.{CRSWorldExtent, LayoutDefinition} import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata} -import geotrellis.spark.tiling.LayoutDefinition -import geotrellis.spark.tiling.CRSWorldExtent +import org.apache.spark.sql.Encoders +import org.locationtech.rasterframes.util.SubdivideSupport /** * Tests miscellaneous extension methods. @@ -32,20 +34,20 @@ import geotrellis.spark.tiling.CRSWorldExtent * @since 3/20/18 */ //noinspection ScalaUnusedSymbol -class ExtensionMethodSpec extends TestEnvironment with TestData { - lazy val rf = sampleTileLayerRDD.toRF +class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSupport { + lazy val rf = sampleTileLayerRDD.toLayer describe("DataFrame exention methods") { it("should maintain original type") { val df = rf.withPrefixedColumnNames("_foo_") - "val rf2: RasterFrame = df" should compile + "val rf2: RasterFrameLayer = df" should compile } it("should provide tagged column access") { val df = rf.drop("tile") "val Some(col) = df.spatialKeyColumn" should compile } } - describe("RasterFrame exention methods") { + describe("RasterFrameLayer exention methods") { it("should provide spatial key column") { noException should be thrownBy { rf.spatialKeyColumn @@ -54,14 +56,28 @@ class ExtensionMethodSpec extends TestEnvironment with TestData { } } describe("Miscellaneous extensions") { + import spark.implicits._ + + it("should find multiple extent columns") { + val df = Seq((extent, "fred", extent, 34.0)).toDF("e1", "s", "e2", "n") + df.extentColumns.size should be(2) + } + + it("should find multiple crs columns") { + // Not sure why implicit resolution isn't handling this properly. + implicit val enc = Encoders.tuple(crsEncoder, Encoders.STRING, crsEncoder, Encoders.scalaDouble) + val df = Seq((pe.crs, "fred", pe.crs, 34.0)).toDF("c1", "s", "c2", "n") + df.crsColumns.size should be(2) + } + it("should split TileLayout") { val tl1 = TileLayout(2, 3, 10, 10) assert(tl1.subdivide(0) === tl1) assert(tl1.subdivide(1) === tl1) assert(tl1.subdivide(2) === TileLayout(4, 6, 5, 5)) assertThrows[IllegalArgumentException](tl1.subdivide(-1)) - } + it("should split KeyBounds[SpatialKey]") { val grid = GridBounds(0, 0, 9, 9) val kb = KeyBounds(grid) @@ -76,10 +92,10 @@ class ExtensionMethodSpec extends TestEnvironment with TestData { it("should split key") { val s1 = SpatialKey(0, 0).subdivide(2) - assert(s1 === Seq(SpatialKey(0,0), SpatialKey(1,0), SpatialKey(0,1), SpatialKey(1,1))) + assert(s1 === Seq(SpatialKey(0, 0), SpatialKey(1, 0), SpatialKey(0, 1), SpatialKey(1, 1))) val s2 = SpatialKey(2, 3).subdivide(3) - assert(s2 === Seq(SpatialKey(6,9), SpatialKey(7,9), SpatialKey(8,9), SpatialKey(6,10), SpatialKey(7,10), SpatialKey(8,10), SpatialKey(6,11), SpatialKey(7,11), SpatialKey(8,11))) + assert(s2 === Seq(SpatialKey(6, 9), SpatialKey(7, 9), SpatialKey(8, 9), SpatialKey(6, 10), SpatialKey(7, 10), SpatialKey(8, 10), SpatialKey(6, 11), SpatialKey(7, 11), SpatialKey(8, 11))) } it("should split TileLayerMetadata[SpatialKey]") { @@ -91,7 +107,12 @@ class ExtensionMethodSpec extends TestEnvironment with TestData { val divided = tlm.subdivide(2) - assert(divided.tileLayout.tileDimensions === (tileSize/2, tileSize/2)) + assert(divided.tileLayout.tileDimensions === (tileSize / 2, tileSize / 2)) + } + + it("should render Markdown") { + import org.locationtech.rasterframes.util._ + rf.toMarkdown().count(_ == '|') shouldBe >=(3 * 5) } } } diff --git a/core/src/test/scala/astraea/spark/rasterframes/JTSSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/GeometryFunctionsSpec.scala similarity index 52% rename from core/src/test/scala/astraea/spark/rasterframes/JTSSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/GeometryFunctionsSpec.scala index 52def8620..54321d0dc 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/JTSSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/GeometryFunctionsSpec.scala @@ -15,22 +15,27 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes -import com.vividsolutions.jts.geom._ import geotrellis.proj4.{LatLng, Sinusoidal, WebMercator} -import geotrellis.vector.{Point ⇒ GTPoint} +import geotrellis.vector.{Extent, Point => GTPoint} +import org.locationtech.jts.geom._ +import spray.json.JsNumber /** * Test rig for operations providing interop with JTS types. * * @since 12/16/17 */ -class JTSSpec extends TestEnvironment with TestData with StandardColumns { - describe("JTS interop") { - val rf = l8Sample(1).projectedRaster.toRF(10, 10).withBounds() +class GeometryFunctionsSpec extends TestEnvironment with TestData with StandardColumns { + import spark.implicits._ + + describe("Vector geometry operations") { + val rf = l8Sample(1).projectedRaster.toLayer(10, 10).withGeometry() it("should allow joining and filtering of tiles based on points") { import spark.implicits._ @@ -43,32 +48,32 @@ class JTSSpec extends TestEnvironment with TestData with StandardColumns { val locs = coords.toDF("id", "point") withClue("join with point column") { - assert(rf.join(locs, st_contains(BOUNDS_COLUMN, $"point")).count === coords.length) - assert(rf.join(locs, st_intersects(BOUNDS_COLUMN, $"point")).count === coords.length) + assert(rf.join(locs, st_contains(GEOMETRY_COLUMN, $"point")).count === coords.length) + assert(rf.join(locs, st_intersects(GEOMETRY_COLUMN, $"point")).count === coords.length) } withClue("point literal") { val point = coords.head._2 - assert(rf.filter(st_contains(BOUNDS_COLUMN, geomLit(point))).count === 1) - assert(rf.filter(st_intersects(BOUNDS_COLUMN, geomLit(point))).count === 1) - assert(rf.filter(BOUNDS_COLUMN intersects point).count === 1) - assert(rf.filter(BOUNDS_COLUMN intersects GTPoint(point)).count === 1) - assert(rf.filter(BOUNDS_COLUMN containsGeom point).count === 1) + assert(rf.filter(st_contains(GEOMETRY_COLUMN, geomLit(point))).count === 1) + assert(rf.filter(st_intersects(GEOMETRY_COLUMN, geomLit(point))).count === 1) + assert(rf.filter(GEOMETRY_COLUMN intersects point).count === 1) + assert(rf.filter(GEOMETRY_COLUMN intersects GTPoint(point)).count === 1) + assert(rf.filter(GEOMETRY_COLUMN containsGeom point).count === 1) } withClue("exercise predicates") { val point = geomLit(coords.head._2) - assert(rf.filter(st_covers(BOUNDS_COLUMN, point)).count === 1) - assert(rf.filter(st_crosses(BOUNDS_COLUMN, point)).count === 0) - assert(rf.filter(st_disjoint(BOUNDS_COLUMN, point)).count === rf.count - 1) - assert(rf.filter(st_overlaps(BOUNDS_COLUMN, point)).count === 0) - assert(rf.filter(st_touches(BOUNDS_COLUMN, point)).count === 0) - assert(rf.filter(st_within(BOUNDS_COLUMN, point)).count === 0) + assert(rf.filter(st_covers(GEOMETRY_COLUMN, point)).count === 1) + assert(rf.filter(st_crosses(GEOMETRY_COLUMN, point)).count === 0) + assert(rf.filter(st_disjoint(GEOMETRY_COLUMN, point)).count === rf.count - 1) + assert(rf.filter(st_overlaps(GEOMETRY_COLUMN, point)).count === 0) + assert(rf.filter(st_touches(GEOMETRY_COLUMN, point)).count === 0) + assert(rf.filter(st_within(GEOMETRY_COLUMN, point)).count === 0) } } it("should allow construction of geometry literals") { - import JTS._ + import GeomData._ assert(dfBlank.select(geomLit(point)).first === point) assert(dfBlank.select(geomLit(line)).first === line) assert(dfBlank.select(geomLit(poly)).first === poly) @@ -80,8 +85,8 @@ class JTSSpec extends TestEnvironment with TestData with StandardColumns { it("should provide a means of getting a bounding box") { import spark.implicits._ - val boxed = rf.select(BOUNDS_COLUMN, envelope(BOUNDS_COLUMN) as "env") - assert(boxed.select($"env".as[Envelope]).first.getArea > 0) + val boxed = rf.select(GEOMETRY_COLUMN, st_extent(GEOMETRY_COLUMN) as "extent") + assert(boxed.select($"extent".as[Extent]).first.area > 0) assert(boxed.toDF("bounds", "bbox").select("bbox.*").schema.length === 4) } @@ -108,9 +113,9 @@ class JTSSpec extends TestEnvironment with TestData with StandardColumns { val df = Seq((latLng, webMercator)).toDF("ll", "wm") val rp = df.select( - reproject_geometry($"ll", LatLng, WebMercator) as "wm2", - reproject_geometry($"wm", WebMercator, LatLng) as "ll2", - reproject_geometry(reproject_geometry($"ll", LatLng, Sinusoidal), Sinusoidal, WebMercator) as "wm3" + st_reproject($"ll", LatLng, WebMercator) as "wm2", + st_reproject($"wm", WebMercator, LatLng) as "ll2", + st_reproject(st_reproject($"ll", LatLng, Sinusoidal), Sinusoidal, WebMercator) as "wm3" ).as[(Geometry, Geometry, Geometry)] @@ -123,9 +128,41 @@ class JTSSpec extends TestEnvironment with TestData with StandardColumns { df.createOrReplaceTempView("geom") - val wm4 = sql("SELECT rf_reproject_geometry(ll, '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', 'EPSG:3857') AS wm4 from geom") + val wm4 = sql("SELECT st_reproject(ll, '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', 'EPSG:3857') AS wm4 from geom") .as[Geometry].first() wm4 should matchGeom(webMercator, 0.00001) + + // TODO: See comment in `org.locationtech.rasterframes.expressions.register` for + // TODO: what needs to happen to support this. + //checkDocs("st_reproject") } } + + it("should rasterize geometry") { + val rf = l8Sample(1).projectedRaster.toLayer.withGeometry() + val df = GeomData.features.map(f ⇒ ( + f.geom.reproject(LatLng, rf.crs).jtsGeom, + f.data.fields("id").asInstanceOf[JsNumber].value.intValue() + )).toDF("geom", "__fid__") + + val toRasterize = rf.crossJoin(df) + + val tlm = rf.tileLayerMetadata.merge + + val (cols, rows) = tlm.layout.tileLayout.tileDimensions + + val rasterized = toRasterize.withColumn("rasterized", rf_rasterize($"geom", GEOMETRY_COLUMN, $"__fid__", cols, rows)) + + assert(rasterized.count() === df.count() * rf.count()) + assert(rasterized.select(rf_dimensions($"rasterized")).distinct().count() === 1) + val pixelCount = rasterized.select(rf_agg_data_cells($"rasterized")).first() + assert(pixelCount < cols * rows) + + + toRasterize.createOrReplaceTempView("stuff") + val viaSQL = sql(s"select rf_rasterize(geom, geometry, __fid__, $cols, $rows) as rasterized from stuff") + assert(viaSQL.select(rf_agg_data_cells($"rasterized")).first === pixelCount) + + //rasterized.select($"rasterized".as[Tile]).foreach(t ⇒ t.renderPng(ColorMaps.IGBP).write("target/" + t.hashCode() + ".png")) + } } diff --git a/core/src/test/scala/astraea/spark/rasterframes/MetadataSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/MetadataSpec.scala similarity index 60% rename from core/src/test/scala/astraea/spark/rasterframes/MetadataSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/MetadataSpec.scala index 4960f7e65..0f179937a 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/MetadataSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/MetadataSpec.scala @@ -1,4 +1,25 @@ -package astraea.spark.rasterframes +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes import org.apache.spark.sql.types.MetadataBuilder @@ -14,7 +35,7 @@ class MetadataSpec extends TestEnvironment with TestData { describe("Metadata storage") { it("should serialize and attach metadata") { - //val rf = sampleGeoTiff.projectedRaster.toRF(128, 128) + //val rf = sampleGeoTiff.projectedRaster.toLayer(128, 128) val df = spark.createDataset(Seq((1, "one"), (2, "two"), (3, "three"))).toDF("num", "str") val withmeta = df.mapColumnAttribute($"num", attr ⇒ { attr.withMetadata(sampleMetadata) diff --git a/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala similarity index 79% rename from core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala index 984da98e7..e77a0fecc 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala @@ -1,11 +1,32 @@ -package astraea.spark.rasterframes +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes import java.sql.Timestamp import java.time.ZonedDateTime -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.util._ import geotrellis.proj4.LatLng import geotrellis.raster.render.{ColorMap, ColorRamp} import geotrellis.raster.{ProjectedRaster, Tile, TileFeature, TileLayout, UByteCellType} @@ -14,11 +35,12 @@ import geotrellis.spark.tiling._ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.functions._ import org.apache.spark.sql.{SQLContext, SparkSession} +import org.locationtech.rasterframes.model.TileDimensions import scala.util.control.NonFatal /** - * RasterFrame test rig. + * RasterFrameLayer test rig. * * @since 7/10/17 */ @@ -52,12 +74,12 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } } - describe("RasterFrame") { + describe("RasterFrameLayer") { it("should implicitly convert from spatial layer type") { val tileLayerRDD = TestData.randomSpatialTileLayerRDD(20, 20, 2, 2) - val rf = tileLayerRDD.toRF + val rf = tileLayerRDD.toLayer assert(rf.tileColumns.nonEmpty) assert(rf.spatialKeyColumn.columnName == "spatial_key") @@ -66,16 +88,14 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys assert(rf.schema.head.metadata.json.contains("tileLayout")) assert( - rf.select(tile_dimensions($"tile")) - .as[Tuple1[(Int, Int)]] - .map(_._1) + rf.select(rf_dimensions($"tile")) .collect() - .forall(_ == (10, 10)) + .forall(_ == TileDimensions(10, 10)) ) assert(rf.count() === 4) - val cols = tileLayerRDD.toRF("foo").columns + val cols = tileLayerRDD.toLayer("foo").columns assert(!cols.contains("tile")) assert(cols.contains("foo")) } @@ -84,7 +104,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys val tileLayerRDD = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2) - val rf = tileLayerRDD.toRF + val rf = tileLayerRDD.toLayer try { assert(rf.tileColumns.nonEmpty) @@ -93,11 +113,10 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } catch { case NonFatal(ex) ⇒ - rf.printSchema() println(rf.schema.prettyJson) throw ex } - val cols = tileLayerRDD.toRF("foo").columns + val cols = tileLayerRDD.toLayer("foo").columns assert(!cols.contains("tile")) assert(cols.contains("foo")) } @@ -116,7 +135,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys val tileLayerRDD = TileFeatureLayerRDD(tileRDD, metadata) - val rf = tileLayerRDD.toRF + val rf = tileLayerRDD.toLayer assert(rf.columns.toSet === Set(SPATIAL_KEY_COLUMN, TILE_COLUMN, TILE_FEATURE_DATA_COLUMN).map(_.columnName)) } @@ -135,14 +154,14 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys val tileLayerRDD = TileFeatureLayerRDD(tileRDD, metadata) - val rf = tileLayerRDD.toRF + val rf = tileLayerRDD.toLayer assert(rf.columns.toSet === Set(SPATIAL_KEY_COLUMN, TEMPORAL_KEY_COLUMN, TILE_COLUMN, TILE_FEATURE_DATA_COLUMN).map(_.columnName)) } it("should support adding a timestamp column") { val now = ZonedDateTime.now() - val rf = sampleGeoTiff.projectedRaster.toRF(256, 256) + val rf = sampleGeoTiff.projectedRaster.toLayer(256, 256) val wt = rf.addTemporalComponent(now) val goodie = wt.withTimestamp() assert(goodie.columns.contains("timestamp")) @@ -153,7 +172,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } it("should support spatial joins") { - val rf = sampleGeoTiff.projectedRaster.toRF(256, 256) + val rf = sampleGeoTiff.projectedRaster.toLayer(256, 256) val wt = rf.addTemporalComponent(TemporalKey(34)) @@ -167,11 +186,11 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } it("should have correct schema on inner spatial joins") { - val left = sampleGeoTiff.projectedRaster.toRF(256, 256) + val left = sampleGeoTiff.projectedRaster.toLayer(256, 256) .addTemporalComponent(TemporalKey(34)) val right = left.withColumnRenamed(left.tileColumns.head.columnName, "rightTile") - .asRF + .asLayer val joined = left.spatialJoin(right) // since right is a copy of left, should not drop any rows with inner join @@ -180,22 +199,25 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys // Should use left's key column names assert(joined.spatialKeyColumn.columnName === left.spatialKeyColumn.columnName) assert(joined.temporalKeyColumn.map(_.columnName) === left.temporalKeyColumn.map(_.columnName)) - + assert(joined.tileColumns.size === 2) + assert(joined.notTileColumns.size === 2) + assert(joined.tileColumns.toSet === joined.tileColumns.toSet) + assert(joined.tileColumns.toSet !== joined.notTileColumns.toSet) } - it("should convert a GeoTiff to RasterFrame") { + it("should convert a GeoTiff to RasterFrameLayer") { val praster: ProjectedRaster[Tile] = sampleGeoTiff.projectedRaster val (cols, rows) = praster.raster.dimensions val layoutCols = math.ceil(cols / 128.0).toInt val layoutRows = math.ceil(rows / 128.0).toInt - assert(praster.toRF.count() === 1) - assert(praster.toRF(128, 128).count() === (layoutCols * layoutRows)) + assert(praster.toLayer.count() === 1) + assert(praster.toLayer(128, 128).count() === (layoutCols * layoutRows)) } it("should provide TileLayerMetadata[SpatialKey]") { - val rf = sampleGeoTiff.projectedRaster.toRF(256, 256) + val rf = sampleGeoTiff.projectedRaster.toLayer(256, 256) val tlm = rf.tileLayerMetadata.merge val bounds = tlm.bounds.get assert(bounds === KeyBounds(SpatialKey(0, 0), SpatialKey(3, 1))) @@ -203,7 +225,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys it("should provide TileLayerMetadata[SpaceTimeKey]") { val now = ZonedDateTime.now() - val rf = sampleGeoTiff.projectedRaster.toRF(256, 256, now) + val rf = sampleGeoTiff.projectedRaster.toLayer(256, 256, now) val tlm = rf.tileLayerMetadata.merge val bounds = tlm.bounds.get assert(bounds._1 === SpaceTimeKey(0, 0, now)) @@ -213,7 +235,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys // it("should clip TileLayerMetadata extent") { // val tiled = sampleTileLayerRDD // -// val rf = tiled.reproject(LatLng, tiled.metadata.layout)._2.toRF +// val rf = tiled.reproject(LatLng, tiled.metadata.layout)._2.toLayer // // val worldish = Extent(-179, -89, 179, 89) // val areaish = Extent(-90, 30, -81, 40) @@ -240,14 +262,14 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } it("shouldn't clip already clipped extents") { - val rf = TestData.randomSpatialTileLayerRDD(1024, 1024, 8, 8).toRF + val rf = TestData.randomSpatialTileLayerRDD(1024, 1024, 8, 8).toLayer val expected = rf.tileLayerMetadata.merge.extent val computed = rf.clipLayerExtent.tileLayerMetadata.merge.extent basicallySame(expected, computed) val pr = sampleGeoTiff.projectedRaster - val rf2 = pr.toRF(256, 256) + val rf2 = pr.toLayer(256, 256) val expected2 = rf2.tileLayerMetadata.merge.extent val computed2 = rf2.clipLayerExtent.tileLayerMetadata.merge.extent basicallySame(expected2, computed2) @@ -272,7 +294,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } it("should rasterize with a spatiotemporal key") { - val rf = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toRF + val rf = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toLayer noException shouldBe thrownBy { rf.toRaster($"tile", 128, 128) @@ -280,8 +302,8 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } it("should maintain metadata after all spatial join operations") { - val rf1 = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toRF - val rf2 = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toRF + val rf1 = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toLayer + val rf2 = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toLayer val joinTypes = Seq("inner", "outer", "fullouter", "left_outer", "right_outer", "leftsemi") forEvery(joinTypes) { jt ⇒ @@ -293,9 +315,9 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys it("should rasterize multiband") { withClue("Landsat") { - val blue = TestData.l8Sample(1).projectedRaster.toRF.withRFColumnRenamed("tile", "blue") - val green = TestData.l8Sample(2).projectedRaster.toRF.withRFColumnRenamed("tile", "green") - val red = TestData.l8Sample(3).projectedRaster.toRF.withRFColumnRenamed("tile", "red") + val blue = TestData.l8Sample(1).projectedRaster.toLayer.withRFColumnRenamed("tile", "blue") + val green = TestData.l8Sample(2).projectedRaster.toLayer.withRFColumnRenamed("tile", "green") + val red = TestData.l8Sample(3).projectedRaster.toLayer.withRFColumnRenamed("tile", "red") val joined = blue.spatialJoin(green).spatialJoin(red) @@ -306,9 +328,9 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys } } withClue("NAIP") { - val red = TestData.naipSample(1).projectedRaster.toRF.withRFColumnRenamed("tile", "red") - val green = TestData.naipSample(2).projectedRaster.toRF.withRFColumnRenamed("tile", "green") - val blue = TestData.naipSample(3).projectedRaster.toRF.withRFColumnRenamed("tile", "blue") + val red = TestData.naipSample(1).projectedRaster.toLayer.withRFColumnRenamed("tile", "red") + val green = TestData.naipSample(2).projectedRaster.toLayer.withRFColumnRenamed("tile", "green") + val blue = TestData.naipSample(3).projectedRaster.toLayer.withRFColumnRenamed("tile", "blue") val joined = blue.spatialJoin(green).spatialJoin(red) noException shouldBe thrownBy { @@ -323,7 +345,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys // 774 × 500 val praster: ProjectedRaster[Tile] = sampleGeoTiff.projectedRaster val (cols, rows) = praster.raster.dimensions - val rf = praster.toRF(64, 64) + val rf = praster.toLayer(64, 64) val raster = rf.toRaster($"tile", cols, rows) render(raster.tile, "normal") @@ -344,7 +366,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys it("shouldn't restitch raster that's has derived tiles") { val praster: ProjectedRaster[Tile] = sampleGeoTiff.projectedRaster - val rf = praster.toRF(64, 64) + val rf = praster.toLayer(64, 64) val equalizer = udf((t: Tile) => t.equalize()) @@ -352,13 +374,13 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys intercept[IllegalArgumentException] { // spatial_key is lost - equalized.asRF.toRaster($"equalized", 128, 128) + equalized.asLayer.toRaster($"equalized", 128, 128) } } it("should fetch CRS") { val praster: ProjectedRaster[Tile] = sampleGeoTiff.projectedRaster - val rf = praster.toRF + val rf = praster.toLayer assert(rf.crs === praster.crs) } diff --git a/core/src/test/scala/astraea/spark/rasterframes/RasterFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala similarity index 55% rename from core/src/test/scala/astraea/spark/rasterframes/RasterFunctionsSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala index da2ab9c56..caccd74ca 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/RasterFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala @@ -19,22 +19,22 @@ * */ -package astraea.spark.rasterframes -import astraea.spark.rasterframes.TestData.injectND -import astraea.spark.rasterframes.expressions.accessors.ExtractTile -import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics} -import astraea.spark.rasterframes.tiles.ProjectedRasterTile +package org.locationtech.rasterframes + import geotrellis.proj4.LatLng import geotrellis.raster import geotrellis.raster.testkit.RasterMatchers -import geotrellis.raster.{ArrayTile, BitCellType, ByteUserDefinedNoDataCellType, DoubleConstantNoDataCellType, ShortConstantNoDataCellType, Tile, UByteConstantNoDataCellType} +import geotrellis.raster._ import geotrellis.vector.Extent -import org.apache.spark.sql.{AnalysisException, Encoders} +import org.apache.spark.sql.Encoders import org.apache.spark.sql.functions._ -import org.scalatest.{FunSpec, Matchers} +import org.locationtech.rasterframes.expressions.accessors.ExtractTile +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.ref.{RasterRef, RasterSource} +import org.locationtech.rasterframes.stats._ +import org.locationtech.rasterframes.tiles.ProjectedRasterTile -class RasterFunctionsSpec extends FunSpec - with TestEnvironment with Matchers with RasterMatchers { +class RasterFunctionsSpec extends TestEnvironment with RasterMatchers { import spark.implicits._ val extent = Extent(10, 20, 30, 40) @@ -51,66 +51,95 @@ class RasterFunctionsSpec extends FunSpec lazy val three = TestData.projectedRasterTile(cols, rows, 3, extent, crs, ct) lazy val six = ProjectedRasterTile(three * two, three.extent, three.crs) lazy val nd = TestData.projectedRasterTile(cols, rows, -2, extent, crs, ct) - lazy val randTile = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextInt(), extent, crs, ct) - lazy val randNDTile = TestData.injectND(numND)(randTile) + lazy val randPRT = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextInt(), extent, crs, ct) + lazy val randNDPRT = TestData.injectND(numND)(randPRT) lazy val randDoubleTile = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextGaussian(), extent, crs, DoubleConstantNoDataCellType) lazy val randDoubleNDTile = TestData.injectND(numND)(randDoubleTile) lazy val randPositiveDoubleTile = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextDouble() + 1e-6, extent, crs, DoubleConstantNoDataCellType) - val expectedRandNoData: Long = numND * tileCount + val expectedRandNoData: Long = numND * tileCount.toLong val expectedRandData: Long = cols * rows * tileCount - expectedRandNoData - lazy val randNDTilesWithNull = Seq.fill[Tile](tileCount)(injectND(numND)( + lazy val randNDTilesWithNull = Seq.fill[Tile](tileCount)(TestData.injectND(numND)( TestData.randomTile(cols, rows, UByteConstantNoDataCellType) )).map(ProjectedRasterTile(_, extent, crs)) :+ null + def lazyPRT = RasterRef(RasterSource(TestData.l8samplePath), 0, None).tile + implicit val pairEnc = Encoders.tuple(ProjectedRasterTile.prtEncoder, ProjectedRasterTile.prtEncoder) implicit val tripEnc = Encoders.tuple(ProjectedRasterTile.prtEncoder, ProjectedRasterTile.prtEncoder, ProjectedRasterTile.prtEncoder) + describe("constant tile generation operations") { + val dim = 2 + val rows = 2 + + it("should create a ones tile") { + val df = (0 until rows).toDF("id") + .withColumn("const", rf_make_ones_tile(dim, dim, IntConstantNoDataCellType)) + val result = df.select(rf_tile_sum($"const") as "ts").agg(sum("ts")).as[Double].first() + result should be (dim * dim * rows) + } + + it("should create a zeros tile") { + val df = (0 until rows).toDF("id") + .withColumn("const", rf_make_zeros_tile(dim, dim, FloatConstantNoDataCellType)) + val result = df.select(rf_tile_sum($"const") as "ts").agg(sum("ts")).as[Double].first() + result should be (0) + } + + it("should create an arbitrary constant tile") { + val value = 4 + val df = (0 until rows).toDF("id") + .withColumn("const", rf_make_constant_tile(value, dim, dim, ByteConstantNoDataCellType)) + val result = df.select(rf_tile_sum($"const") as "ts").agg(sum("ts")).as[Double].first() + result should be (dim * dim * rows * value) + } + } + describe("arithmetic tile operations") { it("should local_add") { val df = Seq((one, two)).toDF("one", "two") - val maybeThree = df.select(local_add($"one", $"two")).as[ProjectedRasterTile] + val maybeThree = df.select(rf_local_add($"one", $"two")).as[ProjectedRasterTile] assertEqual(maybeThree.first(), three) assertEqual(df.selectExpr("rf_local_add(one, two)").as[ProjectedRasterTile].first(), three) - val maybeThreeTile = df.select(local_add(ExtractTile($"one"), ExtractTile($"two"))).as[Tile] + val maybeThreeTile = df.select(rf_local_add(ExtractTile($"one"), ExtractTile($"two"))).as[Tile] assertEqual(maybeThreeTile.first(), three.toArrayTile()) checkDocs("rf_local_add") } - it("should local_subtract") { + it("should rf_local_subtract") { val df = Seq((three, two)).toDF("three", "two") - val maybeOne = df.select(local_subtract($"three", $"two")).as[ProjectedRasterTile] + val maybeOne = df.select(rf_local_subtract($"three", $"two")).as[ProjectedRasterTile] assertEqual(maybeOne.first(), one) assertEqual(df.selectExpr("rf_local_subtract(three, two)").as[ProjectedRasterTile].first(), one) val maybeOneTile = - df.select(local_subtract(ExtractTile($"three"), ExtractTile($"two"))).as[Tile] + df.select(rf_local_subtract(ExtractTile($"three"), ExtractTile($"two"))).as[Tile] assertEqual(maybeOneTile.first(), one.toArrayTile()) checkDocs("rf_local_subtract") } - it("should local_multiply") { + it("should rf_local_multiply") { val df = Seq((three, two)).toDF("three", "two") - val maybeSix = df.select(local_multiply($"three", $"two")).as[ProjectedRasterTile] + val maybeSix = df.select(rf_local_multiply($"three", $"two")).as[ProjectedRasterTile] assertEqual(maybeSix.first(), six) assertEqual(df.selectExpr("rf_local_multiply(three, two)").as[ProjectedRasterTile].first(), six) val maybeSixTile = - df.select(local_multiply(ExtractTile($"three"), ExtractTile($"two"))).as[Tile] + df.select(rf_local_multiply(ExtractTile($"three"), ExtractTile($"two"))).as[Tile] assertEqual(maybeSixTile.first(), six.toArrayTile()) checkDocs("rf_local_multiply") } - it("should local_divide") { + it("should rf_local_divide") { val df = Seq((six, two)).toDF("six", "two") - val maybeThree = df.select(local_divide($"six", $"two")).as[ProjectedRasterTile] + val maybeThree = df.select(rf_local_divide($"six", $"two")).as[ProjectedRasterTile] assertEqual(maybeThree.first(), three) assertEqual(df.selectExpr("rf_local_divide(six, two)").as[ProjectedRasterTile].first(), three) @@ -119,151 +148,176 @@ class RasterFunctionsSpec extends FunSpec .as[ProjectedRasterTile].first(), six) val maybeThreeTile = - df.select(local_divide(ExtractTile($"six"), ExtractTile($"two"))).as[Tile] + df.select(rf_local_divide(ExtractTile($"six"), ExtractTile($"two"))).as[Tile] assertEqual(maybeThreeTile.first(), three.toArrayTile()) checkDocs("rf_local_divide") } } describe("scalar tile operations") { - it("should local_add") { + it("should rf_local_add") { val df = Seq(one).toDF("one") - val maybeThree = df.select(local_add($"one", 2)).as[ProjectedRasterTile] + val maybeThree = df.select(rf_local_add($"one", 2)).as[ProjectedRasterTile] assertEqual(maybeThree.first(), three) - val maybeThreeD = df.select(local_add($"one", 2.1)).as[ProjectedRasterTile] + val maybeThreeD = df.select(rf_local_add($"one", 2.1)).as[ProjectedRasterTile] assertEqual(maybeThreeD.first(), three.convert(DoubleConstantNoDataCellType).localAdd(0.1)) - val maybeThreeTile = df.select(local_add(ExtractTile($"one"), 2)).as[Tile] + val maybeThreeTile = df.select(rf_local_add(ExtractTile($"one"), 2)).as[Tile] assertEqual(maybeThreeTile.first(), three.toArrayTile()) } - it("should local_subtract") { + it("should rf_local_subtract") { val df = Seq(three).toDF("three") - val maybeOne = df.select(local_subtract($"three", 2)).as[ProjectedRasterTile] + val maybeOne = df.select(rf_local_subtract($"three", 2)).as[ProjectedRasterTile] assertEqual(maybeOne.first(), one) - val maybeOneD = df.select(local_subtract($"three", 2.0)).as[ProjectedRasterTile] + val maybeOneD = df.select(rf_local_subtract($"three", 2.0)).as[ProjectedRasterTile] assertEqual(maybeOneD.first(), one) - val maybeOneTile = df.select(local_subtract(ExtractTile($"three"), 2)).as[Tile] + val maybeOneTile = df.select(rf_local_subtract(ExtractTile($"three"), 2)).as[Tile] assertEqual(maybeOneTile.first(), one.toArrayTile()) } - it("should local_multiply") { + it("should rf_local_multiply") { val df = Seq(three).toDF("three") - val maybeSix = df.select(local_multiply($"three", 2)).as[ProjectedRasterTile] + val maybeSix = df.select(rf_local_multiply($"three", 2)).as[ProjectedRasterTile] assertEqual(maybeSix.first(), six) - val maybeSixD = df.select(local_multiply($"three", 2.0)).as[ProjectedRasterTile] + val maybeSixD = df.select(rf_local_multiply($"three", 2.0)).as[ProjectedRasterTile] assertEqual(maybeSixD.first(), six) - val maybeSixTile = df.select(local_multiply(ExtractTile($"three"), 2)).as[Tile] + val maybeSixTile = df.select(rf_local_multiply(ExtractTile($"three"), 2)).as[Tile] assertEqual(maybeSixTile.first(), six.toArrayTile()) } - it("should local_divide") { + it("should rf_local_divide") { val df = Seq(six).toDF("six") - val maybeThree = df.select(local_divide($"six", 2)).as[ProjectedRasterTile] + val maybeThree = df.select(rf_local_divide($"six", 2)).as[ProjectedRasterTile] assertEqual(maybeThree.first(), three) - val maybeThreeD = df.select(local_divide($"six", 2.0)).as[ProjectedRasterTile] + val maybeThreeD = df.select(rf_local_divide($"six", 2.0)).as[ProjectedRasterTile] assertEqual(maybeThreeD.first(), three) - val maybeThreeTile = df.select(local_divide(ExtractTile($"six"), 2)).as[Tile] + val maybeThreeTile = df.select(rf_local_divide(ExtractTile($"six"), 2)).as[Tile] assertEqual(maybeThreeTile.first(), three.toArrayTile()) } } describe("tile comparison relations") { - it("should evaluate local_less") { + it("should evaluate rf_local_less") { val df = Seq((two, three, six)).toDF("two", "three", "six") - df.select(tile_sum(local_less($"two", 6))).first() should be(100.0) - df.select(tile_sum(local_less($"two", 1.9))).first() should be(0.0) - df.select(tile_sum(local_less($"two", 2))).first() should be(0.0) - df.select(tile_sum(local_less($"three", $"two"))).first() should be(0.0) - df.select(tile_sum(local_less($"three", $"three"))).first() should be(0.0) - df.select(tile_sum(local_less($"three", $"six"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less($"two", 6))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less($"two", 1.9))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less($"two", 2))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less($"three", $"two"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less($"three", $"three"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less($"three", $"six"))).first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_less(two, 6))").as[Double].first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_less(three, three))").as[Double].first() should be(0.0) checkDocs("rf_local_less") } - it("should evaluate local_less_equal") { + it("should evaluate rf_local_less_equal") { val df = Seq((two, three, six)).toDF("two", "three", "six") - df.select(tile_sum(local_less_equal($"two", 6))).first() should be(100.0) - df.select(tile_sum(local_less_equal($"two", 1.9))).first() should be(0.0) - df.select(tile_sum(local_less_equal($"two", 2))).first() should be(100.0) - df.select(tile_sum(local_less_equal($"three", $"two"))).first() should be(0.0) - df.select(tile_sum(local_less_equal($"three", $"three"))).first() should be(100.0) - df.select(tile_sum(local_less_equal($"three", $"six"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less_equal($"two", 6))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less_equal($"two", 1.9))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less_equal($"two", 2))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less_equal($"three", $"two"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_less_equal($"three", $"three"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_less_equal($"three", $"six"))).first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_less_equal(two, 6))").as[Double].first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_less_equal(three, three))").as[Double].first() should be(100.0) checkDocs("rf_local_less_equal") } - it("should evaluate local_greater") { + it("should evaluate rf_local_greater") { val df = Seq((two, three, six)).toDF("two", "three", "six") - df.select(tile_sum(local_greater($"two", 6))).first() should be(0.0) - df.select(tile_sum(local_greater($"two", 1.9))).first() should be(100.0) - df.select(tile_sum(local_greater($"two", 2))).first() should be(0.0) - df.select(tile_sum(local_greater($"three", $"two"))).first() should be(100.0) - df.select(tile_sum(local_greater($"three", $"three"))).first() should be(0.0) - df.select(tile_sum(local_greater($"three", $"six"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater($"two", 6))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater($"two", 1.9))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater($"two", 2))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater($"three", $"two"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater($"three", $"three"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater($"three", $"six"))).first() should be(0.0) df.selectExpr("rf_tile_sum(rf_local_greater(two, 1.9))").as[Double].first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_greater(three, three))").as[Double].first() should be(0.0) checkDocs("rf_local_greater") } - it("should evaluate local_greater_equal") { + it("should evaluate rf_local_greater_equal") { val df = Seq((two, three, six)).toDF("two", "three", "six") - df.select(tile_sum(local_greater_equal($"two", 6))).first() should be(0.0) - df.select(tile_sum(local_greater_equal($"two", 1.9))).first() should be(100.0) - df.select(tile_sum(local_greater_equal($"two", 2))).first() should be(100.0) - df.select(tile_sum(local_greater_equal($"three", $"two"))).first() should be(100.0) - df.select(tile_sum(local_greater_equal($"three", $"three"))).first() should be(100.0) - df.select(tile_sum(local_greater_equal($"three", $"six"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater_equal($"two", 6))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_greater_equal($"two", 1.9))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater_equal($"two", 2))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater_equal($"three", $"two"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater_equal($"three", $"three"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_greater_equal($"three", $"six"))).first() should be(0.0) df.selectExpr("rf_tile_sum(rf_local_greater_equal(two, 1.9))").as[Double].first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_greater_equal(three, three))").as[Double].first() should be(100.0) checkDocs("rf_local_greater_equal") } - it("should evaluate local_equal") { + it("should evaluate rf_local_equal") { val df = Seq((two, three, three)).toDF("two", "threeA", "threeB") - df.select(tile_sum(local_equal($"two", 2))).first() should be(100.0) - df.select(tile_sum(local_equal($"two", 2.1))).first() should be(0.0) - df.select(tile_sum(local_equal($"two", $"threeA"))).first() should be(0.0) - df.select(tile_sum(local_equal($"threeA", $"threeB"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_equal($"two", 2))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_equal($"two", 2.1))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_equal($"two", $"threeA"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_equal($"threeA", $"threeB"))).first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_equal(two, 1.9))").as[Double].first() should be(0.0) df.selectExpr("rf_tile_sum(rf_local_equal(threeA, threeB))").as[Double].first() should be(100.0) checkDocs("rf_local_equal") } - it("should evaluate local_unequal") { + it("should evaluate rf_local_unequal") { val df = Seq((two, three, three)).toDF("two", "threeA", "threeB") - df.select(tile_sum(local_unequal($"two", 2))).first() should be(0.0) - df.select(tile_sum(local_unequal($"two", 2.1))).first() should be(100.0) - df.select(tile_sum(local_unequal($"two", $"threeA"))).first() should be(100.0) - df.select(tile_sum(local_unequal($"threeA", $"threeB"))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_unequal($"two", 2))).first() should be(0.0) + df.select(rf_tile_sum(rf_local_unequal($"two", 2.1))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_unequal($"two", $"threeA"))).first() should be(100.0) + df.select(rf_tile_sum(rf_local_unequal($"threeA", $"threeB"))).first() should be(0.0) df.selectExpr("rf_tile_sum(rf_local_unequal(two, 1.9))").as[Double].first() should be(100.0) df.selectExpr("rf_tile_sum(rf_local_unequal(threeA, threeB))").as[Double].first() should be(0.0) checkDocs("rf_local_unequal") } } + describe("raster metadata") { + it("should get the TileDimensions of a Tile") { + val t = Seq(randPRT).toDF("tile").select(rf_dimensions($"tile")).first() + t should be (TileDimensions(randPRT.dimensions)) + checkDocs("rf_dimensions") + } + it("should get the Extent of a ProjectedRasterTile") { + val e = Seq(randPRT).toDF("tile").select(rf_extent($"tile")).first() + e should be (extent) + checkDocs("rf_extent") + } + + it("should get the CRS of a ProjectedRasterTile") { + val e = Seq(randPRT).toDF("tile").select(rf_crs($"tile")).first() + e should be (crs) + checkDocs("rf_crs") + } + + it("should get the Geometry of a ProjectedRasterTile") { + val g = Seq(randPRT).toDF("tile").select(rf_geometry($"tile")).first() + g should be (extent.jtsGeom) + checkDocs("rf_geometry") + } + } + describe("per-tile stats") { it("should compute data cell counts") { val df = Seq(TestData.injectND(numND)(two)).toDF("two") - df.select(data_cells($"two")).first() shouldBe (cols * rows - numND).toLong + df.select(rf_data_cells($"two")).first() shouldBe (cols * rows - numND).toLong val df2 = randNDTilesWithNull.toDF("tile") - df2.select(data_cells($"tile") as "cells") + df2.select(rf_data_cells($"tile") as "cells") .agg(sum("cells")) .as[Long] .first() should be (expectedRandData) @@ -272,10 +326,10 @@ class RasterFunctionsSpec extends FunSpec } it("should compute no-data cell counts") { val df = Seq(TestData.injectND(numND)(two)).toDF("two") - df.select(no_data_cells($"two")).first() should be(numND) + df.select(rf_no_data_cells($"two")).first() should be(numND) val df2 = randNDTilesWithNull.toDF("tile") - df2.select(no_data_cells($"tile") as "cells") + df2.select(rf_no_data_cells($"tile") as "cells") .agg(sum("cells")) .as[Long] .first() should be (expectedRandNoData) @@ -284,40 +338,56 @@ class RasterFunctionsSpec extends FunSpec } it("should detect no-data tiles") { val df = Seq(nd).toDF("nd") - df.select(is_no_data_tile($"nd")).first() should be(true) + df.select(rf_is_no_data_tile($"nd")).first() should be(true) val df2 = Seq(two).toDF("not_nd") - df2.select(is_no_data_tile($"not_nd")).first() should be(false) + df2.select(rf_is_no_data_tile($"not_nd")).first() should be(false) checkDocs("rf_is_no_data_tile") } + + it("should evaluate exists and for_all") { + val df0 = Seq(zero).toDF("tile") + df0.select(rf_exists($"tile")).first() should be(false) + df0.select(rf_for_all($"tile")).first() should be(false) + + Seq(one).toDF("tile").select(rf_exists($"tile")).first() should be(true) + Seq(one).toDF("tile").select(rf_for_all($"tile")).first() should be(true) + + val dfNd = Seq(TestData.injectND(1)(one)).toDF("tile") + dfNd.select(rf_exists($"tile")).first() should be(true) + dfNd.select(rf_for_all($"tile")).first() should be(false) + + checkDocs("rf_exists") + checkDocs("rf_for_all") + } it("should find the minimum cell value") { - val min = randNDTile.toArray().filter(c => raster.isData(c)).min.toDouble - val df = Seq(randNDTile).toDF("rand") - df.select(tile_min($"rand")).first() should be(min) + val min = randNDPRT.toArray().filter(c => raster.isData(c)).min.toDouble + val df = Seq(randNDPRT).toDF("rand") + df.select(rf_tile_min($"rand")).first() should be(min) df.selectExpr("rf_tile_min(rand)").as[Double].first() should be(min) checkDocs("rf_tile_min") } it("should find the maximum cell value") { - val max = randNDTile.toArray().filter(c => raster.isData(c)).max.toDouble - val df = Seq(randNDTile).toDF("rand") - df.select(tile_max($"rand")).first() should be(max) + val max = randNDPRT.toArray().filter(c => raster.isData(c)).max.toDouble + val df = Seq(randNDPRT).toDF("rand") + df.select(rf_tile_max($"rand")).first() should be(max) df.selectExpr("rf_tile_max(rand)").as[Double].first() should be(max) checkDocs("rf_tile_max") } it("should compute the tile mean cell value") { - val values = randNDTile.toArray().filter(c => raster.isData(c)) + val values = randNDPRT.toArray().filter(c => raster.isData(c)) val mean = values.sum.toDouble / values.length - val df = Seq(randNDTile).toDF("rand") - df.select(tile_mean($"rand")).first() should be(mean) + val df = Seq(randNDPRT).toDF("rand") + df.select(rf_tile_mean($"rand")).first() should be(mean) df.selectExpr("rf_tile_mean(rand)").as[Double].first() should be(mean) checkDocs("rf_tile_mean") } it("should compute the tile summary statistics") { - val values = randNDTile.toArray().filter(c => raster.isData(c)) + val values = randNDPRT.toArray().filter(c => raster.isData(c)) val mean = values.sum.toDouble / values.length - val df = Seq(randNDTile).toDF("rand") - val stats = df.select(tile_stats($"rand")).first() + val df = Seq(randNDPRT).toDF("rand") + val stats = df.select(rf_tile_stats($"rand")).first() stats.mean should be (mean +- 0.00001) val stats2 = df.selectExpr("rf_tile_stats(rand) as stats") @@ -325,7 +395,7 @@ class RasterFunctionsSpec extends FunSpec .first() stats2 should be (stats) - df.select(tile_stats($"rand") as "stats") + df.select(rf_tile_stats($"rand") as "stats") .select($"stats.mean").as[Double] .first() should be(mean +- 0.00001) df.selectExpr("rf_tile_stats(rand) as stats") @@ -334,7 +404,7 @@ class RasterFunctionsSpec extends FunSpec val df2 = randNDTilesWithNull.toDF("tile") df2 - .select(tile_stats($"tile")("data_cells") as "cells") + .select(rf_tile_stats($"tile")("data_cells") as "cells") .agg(sum("cells")) .as[Long] .first() should be (expectedRandData) @@ -343,8 +413,8 @@ class RasterFunctionsSpec extends FunSpec } it("should compute the tile histogram") { - val df = Seq(randNDTile).toDF("rand") - val h1 = df.select(tile_histogram($"rand")).first() + val df = Seq(randNDPRT).toDF("rand") + val h1 = df.select(rf_tile_histogram($"rand")).first() val h2 = df.selectExpr("rf_tile_histogram(rand) as hist") .select($"hist".as[CellHistogram]) @@ -359,14 +429,14 @@ class RasterFunctionsSpec extends FunSpec describe("aggregate statistics") { it("should count data cells") { val df = randNDTilesWithNull.filter(_ != null).toDF("tile") - df.select(agg_data_cells($"tile")).first() should be (expectedRandData) + df.select(rf_agg_data_cells($"tile")).first() should be (expectedRandData) df.selectExpr("rf_agg_data_cells(tile)").as[Long].first() should be (expectedRandData) checkDocs("rf_agg_data_cells") } it("should count no-data cells") { val df = randNDTilesWithNull.toDF("tile") - df.select(agg_no_data_cells($"tile")).first() should be (expectedRandNoData) + df.select(rf_agg_no_data_cells($"tile")).first() should be (expectedRandNoData) df.selectExpr("rf_agg_no_data_cells(tile)").as[Long].first() should be (expectedRandNoData) checkDocs("rf_agg_no_data_cells") } @@ -375,7 +445,7 @@ class RasterFunctionsSpec extends FunSpec val df = randNDTilesWithNull.toDF("tile") df - .select(agg_stats($"tile") as "stats") + .select(rf_agg_stats($"tile") as "stats") .select("stats.data_cells", "stats.no_data_cells") .as[(Long, Long)] .first() should be ((expectedRandData, expectedRandNoData)) @@ -389,7 +459,7 @@ class RasterFunctionsSpec extends FunSpec it("should compute a aggregate histogram") { val df = randNDTilesWithNull.toDF("tile") - val hist1 = df.select(agg_approx_histogram($"tile")).first() + val hist1 = df.select(rf_agg_approx_histogram($"tile")).first() val hist2 = df.selectExpr("rf_agg_approx_histogram(tile) as hist") .select($"hist".as[CellHistogram]) .first() @@ -399,7 +469,7 @@ class RasterFunctionsSpec extends FunSpec it("should compute local statistics") { val df = randNDTilesWithNull.toDF("tile") - val stats1 = df.select(agg_local_stats($"tile")) + val stats1 = df.select(rf_agg_local_stats($"tile")) .first() val stats2 = df.selectExpr("rf_agg_local_stats(tile) as stats") .select($"stats".as[LocalCellStatistics]) @@ -411,42 +481,42 @@ class RasterFunctionsSpec extends FunSpec it("should compute local min") { val df = Seq(two, three, one, six).toDF("tile") - df.select(agg_local_min($"tile")).first() should be(one.toArrayTile()) + df.select(rf_agg_local_min($"tile")).first() should be(one.toArrayTile()) df.selectExpr("rf_agg_local_min(tile)").as[Tile].first() should be(one.toArrayTile()) checkDocs("rf_agg_local_min") } it("should compute local max") { val df = Seq(two, three, one, six).toDF("tile") - df.select(agg_local_max($"tile")).first() should be(six.toArrayTile()) + df.select(rf_agg_local_max($"tile")).first() should be(six.toArrayTile()) df.selectExpr("rf_agg_local_max(tile)").as[Tile].first() should be(six.toArrayTile()) checkDocs("rf_agg_local_max") } it("should compute local data cell counts") { - val df = Seq(two, randNDTile, nd).toDF("tile") - val t1 = df.select(agg_local_data_cells($"tile")).first() + val df = Seq(two, randNDPRT, nd).toDF("tile") + val t1 = df.select(rf_agg_local_data_cells($"tile")).first() val t2 = df.selectExpr("rf_agg_local_data_cells(tile) as cnt").select($"cnt".as[Tile]).first() t1 should be (t2) checkDocs("rf_agg_local_data_cells") } it("should compute local no-data cell counts") { - val df = Seq(two, randNDTile, nd).toDF("tile") - val t1 = df.select(agg_local_no_data_cells($"tile")).first() + val df = Seq(two, randNDPRT, nd).toDF("tile") + val t1 = df.select(rf_agg_local_no_data_cells($"tile")).first() val t2 = df.selectExpr("rf_agg_local_no_data_cells(tile) as cnt").select($"cnt".as[Tile]).first() t1 should be (t2) - val t3 = df.select(local_add(agg_local_data_cells($"tile"), agg_local_no_data_cells($"tile"))).first() + val t3 = df.select(rf_local_add(rf_agg_local_data_cells($"tile"), rf_agg_local_no_data_cells($"tile"))).first() t3 should be(three.toArrayTile()) checkDocs("rf_agg_local_no_data_cells") } } describe("analytical transformations") { - it("should compute normalized_difference") { + it("should compute rf_normalized_difference") { val df = Seq((three, two)).toDF("three", "two") - df.select(tile_to_array_double(normalized_difference($"three", $"two"))) + df.select(rf_tile_to_array_double(rf_normalized_difference($"three", $"two"))) .first() .forall(_ == 0.2) shouldBe true @@ -459,18 +529,18 @@ class RasterFunctionsSpec extends FunSpec } it("should mask one tile against another") { - val df = Seq[Tile](randTile).toDF("tile") + val df = Seq[Tile](randPRT).toDF("tile") val withMask = df.withColumn("mask", - convert_cell_type( - local_greater($"tile", 50), + rf_convert_cell_type( + rf_local_greater($"tile", 50), "uint8") ) val withMasked = withMask.withColumn("masked", - mask($"tile", $"mask")) + rf_mask($"tile", $"mask")) - val result = withMasked.agg(agg_no_data_cells($"tile") < agg_no_data_cells($"masked")).as[Boolean] + val result = withMasked.agg(rf_agg_no_data_cells($"tile") < rf_agg_no_data_cells($"masked")).as[Boolean] result.first() should be(true) @@ -478,22 +548,22 @@ class RasterFunctionsSpec extends FunSpec } it("should inverse mask one tile against another") { - val df = Seq[Tile](randTile).toDF("tile") + val df = Seq[Tile](randPRT).toDF("tile") - val baseND = df.select(agg_no_data_cells($"tile")).first() + val baseND = df.select(rf_agg_no_data_cells($"tile")).first() val withMask = df.withColumn("mask", - convert_cell_type( - local_greater($"tile", 50), + rf_convert_cell_type( + rf_local_greater($"tile", 50), "uint8" ) ) val withMasked = withMask - .withColumn("masked", mask($"tile", $"mask")) - .withColumn("inv_masked", inverse_mask($"tile", $"mask")) + .withColumn("masked", rf_mask($"tile", $"mask")) + .withColumn("inv_masked", rf_inverse_mask($"tile", $"mask")) - val result = withMasked.agg(agg_no_data_cells($"masked") + agg_no_data_cells($"inv_masked")).as[Long] + val result = withMasked.agg(rf_agg_no_data_cells($"masked") + rf_agg_no_data_cells($"inv_masked")).as[Long] result.first() should be(tileSize + baseND) @@ -501,29 +571,50 @@ class RasterFunctionsSpec extends FunSpec } it("should mask tile by another identified by specified value") { - val df = Seq[Tile](randTile).toDF("tile") + val df = Seq[Tile](randPRT).toDF("tile") val mask_value = 4 val withMask = df.withColumn("mask", - local_multiply(convert_cell_type( - local_greater($"tile", 50), + rf_local_multiply(rf_convert_cell_type( + rf_local_greater($"tile", 50), "uint8"), lit(mask_value) ) ) val withMasked = withMask.withColumn("masked", - mask_by_value($"tile", $"mask", lit(mask_value))) + rf_mask_by_value($"tile", $"mask", lit(mask_value))) - val result = withMasked.agg(agg_no_data_cells($"tile") < agg_no_data_cells($"masked")).as[Boolean] + val result = withMasked.agg(rf_agg_no_data_cells($"tile") < rf_agg_no_data_cells($"masked")).as[Boolean] result.first() should be(true) checkDocs("rf_mask_by_value") } + it("should inverse mask tile by another identified by specified value") { + val df = Seq[Tile](randPRT).toDF("tile") + val mask_value = 4 + + val withMask = df.withColumn("mask", + rf_local_multiply(rf_convert_cell_type( + rf_local_greater($"tile", 50), + "uint8"), + lit(mask_value) + ) + ) + + val withMasked = withMask.withColumn("masked", + rf_inverse_mask_by_value($"tile", $"mask", lit(mask_value))) + + val result = withMasked.agg(rf_agg_no_data_cells($"tile") < rf_agg_no_data_cells($"masked")).as[Boolean] + + result.first() should be(true) + checkDocs("rf_inverse_mask_by_value") + } + it("should render ascii art") { val df = Seq[Tile](ProjectedRasterTile(TestData.l8Labels)).toDF("tile") - val r1 = df.select(render_ascii($"tile")) + val r1 = df.select(rf_render_ascii($"tile")) val r2 = df.selectExpr("rf_render_ascii(tile)").as[String] r1.first() should be(r2.first()) checkDocs("rf_render_ascii") @@ -531,7 +622,7 @@ class RasterFunctionsSpec extends FunSpec it("should render cells as matrix") { val df = Seq(randDoubleNDTile).toDF("tile") - val r1 = df.select(render_matrix($"tile")) + val r1 = df.select(rf_render_matrix($"tile")) val r2 = df.selectExpr("rf_render_matrix(tile)").as[String] r1.first() should be(r2.first()) checkDocs("rf_render_matrix") @@ -545,9 +636,9 @@ class RasterFunctionsSpec extends FunSpec val df = Seq((three_plus, three_less, three)).toDF("three_plus", "three_less", "three") - assertEqual(df.select(round($"three")).as[ProjectedRasterTile].first(), three) - assertEqual(df.select(round($"three_plus")).as[ProjectedRasterTile].first(), three_double) - assertEqual(df.select(round($"three_less")).as[ProjectedRasterTile].first(), three_double) + assertEqual(df.select(rf_round($"three")).as[ProjectedRasterTile].first(), three) + assertEqual(df.select(rf_round($"three_plus")).as[ProjectedRasterTile].first(), three_double) + assertEqual(df.select(rf_round($"three_less")).as[ProjectedRasterTile].first(), three_double) assertEqual(df.selectExpr("rf_round(three)").as[ProjectedRasterTile].first(), three) assertEqual(df.selectExpr("rf_round(three_plus)").as[ProjectedRasterTile].first(), three_double) @@ -556,27 +647,36 @@ class RasterFunctionsSpec extends FunSpec checkDocs("rf_round") } + it("should abs cell values") { + val minus = one.mapTile(t => t.convert(IntConstantNoDataCellType) * -1) + val df = Seq((minus, one)).toDF("minus", "one") + + assertEqual(df.select(rf_abs($"minus").as[ProjectedRasterTile]).first(), one) + + checkDocs("rf_abs") + } + it("should take logarithms positive cell values"){ - // log10 1000 == 3 + // rf_log10 1000 == 3 val thousand = TestData.projectedRasterTile(cols, rows, 1000, extent, crs, ShortConstantNoDataCellType) val threesDouble = TestData.projectedRasterTile(cols, rows, 3.0, extent, crs, DoubleConstantNoDataCellType) val zerosDouble = TestData.projectedRasterTile(cols, rows, 0.0, extent, crs, DoubleConstantNoDataCellType) val df1 = Seq(thousand).toDF("tile") - assertEqual(df1.select(log10($"tile")).as[ProjectedRasterTile].first(), threesDouble) + assertEqual(df1.select(rf_log10($"tile")).as[ProjectedRasterTile].first(), threesDouble) - // ln random tile == log10 random tile / log10(e); random tile square to ensure all positive cell values + // ln random tile == rf_log10 random tile / rf_log10(e); random tile square to ensure all positive cell values val df2 = Seq(randPositiveDoubleTile).toDF("tile") val log10e = math.log10(math.E) - assertEqual(df2.select(log($"tile")).as[ProjectedRasterTile].first(), - df2.select(log10($"tile")).as[ProjectedRasterTile].first() / log10e) + assertEqual(df2.select(rf_log($"tile")).as[ProjectedRasterTile].first(), + df2.select(rf_log10($"tile")).as[ProjectedRasterTile].first() / log10e) lazy val maybeZeros = df2 .selectExpr(s"rf_local_subtract(rf_log(tile), rf_local_divide(rf_log10(tile), ${log10e}))") .as[ProjectedRasterTile].first() assertEqual(maybeZeros, zerosDouble) - // log1p for zeros should be ln(1) + // rf_log1p for zeros should be ln(1) val ln1 = math.log1p(0.0) val df3 = Seq(zero).toDF("tile") val maybeLn1 = df3.selectExpr(s"rf_log1p(tile)").as[ProjectedRasterTile].first() @@ -594,42 +694,42 @@ class RasterFunctionsSpec extends FunSpec // tile zeros ==> -Infinity val df_0 = Seq(zero).toDF("tile") - assertEqual(df_0.select(log($"tile")).as[ProjectedRasterTile].first(), ni_float) - assertEqual(df_0.select(log10($"tile")).as[ProjectedRasterTile].first(), ni_float) - assertEqual(df_0.select(log2($"tile")).as[ProjectedRasterTile].first(), ni_float) - // log1p of zeros should be 0. - assertEqual(df_0.select(log1p($"tile")).as[ProjectedRasterTile].first(), zero_float) + assertEqual(df_0.select(rf_log($"tile")).as[ProjectedRasterTile].first(), ni_float) + assertEqual(df_0.select(rf_log10($"tile")).as[ProjectedRasterTile].first(), ni_float) + assertEqual(df_0.select(rf_log2($"tile")).as[ProjectedRasterTile].first(), ni_float) + // rf_log1p of zeros should be 0. + assertEqual(df_0.select(rf_log1p($"tile")).as[ProjectedRasterTile].first(), zero_float) // tile negative values ==> NaN assert(df_0.selectExpr("rf_log(rf_local_subtract(tile, 42))").as[ProjectedRasterTile].first().isNoDataTile) assert(df_0.selectExpr("rf_log2(rf_local_subtract(tile, 42))").as[ProjectedRasterTile].first().isNoDataTile) - assert(df_0.select(log1p(local_subtract($"tile", 42))).as[ProjectedRasterTile].first().isNoDataTile) - assert(df_0.select(log10(local_subtract($"tile", lit(0.01)))).as[ProjectedRasterTile].first().isNoDataTile) + assert(df_0.select(rf_log1p(rf_local_subtract($"tile", 42))).as[ProjectedRasterTile].first().isNoDataTile) + assert(df_0.select(rf_log10(rf_local_subtract($"tile", lit(0.01)))).as[ProjectedRasterTile].first().isNoDataTile) } it("should take exponential") { val df = Seq(six).toDF("tile") - // exp inverses log + // rf_exp inverses rf_log assertEqual( - df.select(exp(log($"tile"))).as[ProjectedRasterTile].first(), + df.select(rf_exp(rf_log($"tile"))).as[ProjectedRasterTile].first(), six ) // base 2 assertEqual( - df.select(exp2(log2($"tile"))).as[ProjectedRasterTile].first(), + df.select(rf_exp2(rf_log2($"tile"))).as[ProjectedRasterTile].first(), six) // base 10 assertEqual( - df.select(exp10(log10($"tile"))).as[ProjectedRasterTile].first(), + df.select(rf_exp10(rf_log10($"tile"))).as[ProjectedRasterTile].first(), six) // plus/minus 1 assertEqual( - df.select(expm1(log1p($"tile"))).as[ProjectedRasterTile].first(), + df.select(rf_expm1(rf_log1p($"tile"))).as[ProjectedRasterTile].first(), six) // SQL @@ -647,7 +747,7 @@ class RasterFunctionsSpec extends FunSpec df.selectExpr("rf_exp2(rf_log2(tile))").as[ProjectedRasterTile].first(), six) - // SQL expm1 + // SQL rf_expm1 assertEqual( df.selectExpr("rf_expm1(rf_log1p(tile))").as[ProjectedRasterTile].first(), six) @@ -678,11 +778,11 @@ class RasterFunctionsSpec extends FunSpec def df = Seq(lowRes).toDF("tile") - val maybeUp = df.select(resample($"tile", lit(2))).as[ProjectedRasterTile].first() + val maybeUp = df.select(rf_resample($"tile", lit(2))).as[ProjectedRasterTile].first() assertEqual(maybeUp, upsampled) def df2 = Seq((lowRes, fourByFour)).toDF("tile1", "tile2") - val maybeUpShape = df2.select(resample($"tile1", $"tile2")).as[ProjectedRasterTile].first() + val maybeUpShape = df2.select(rf_resample($"tile1", $"tile2")).as[ProjectedRasterTile].first() assertEqual(maybeUpShape, upsampled) // Downsample by double argument < 1 diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterJoinSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterJoinSpec.scala new file mode 100644 index 000000000..b2cd5d8ce --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterJoinSpec.scala @@ -0,0 +1,168 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes + +import geotrellis.raster.resample.Bilinear +import geotrellis.raster.testkit.RasterMatchers +import geotrellis.raster.{IntConstantNoDataCellType, Raster, Tile} +import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate +import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate.ProjectedRasterDefinition +import org.locationtech.rasterframes.model.TileDimensions + + +class RasterJoinSpec extends TestEnvironment with TestData with RasterMatchers { + import spark.implicits._ + describe("Raster join between two DataFrames") { + val b4nativeTif = readSingleband("L8-B4-Elkton-VA.tiff") + // Same data, reprojected to EPSG:4326 + val b4warpedTif = readSingleband("L8-B4-Elkton-VA-4326.tiff") + + val b4nativeRf = b4nativeTif.toDF(TileDimensions(10, 10)) + val b4warpedRf = b4warpedTif.toDF(TileDimensions(10, 10)) + .withColumnRenamed("tile", "tile2") + + it("should join the same scene correctly") { + + val b4nativeRfPrime = b4nativeTif.toDF(TileDimensions(10, 10)) + .withColumnRenamed("tile", "tile2") + val joined = b4nativeRf.rasterJoin(b4nativeRfPrime) + + joined.count() should be (b4nativeRf.count()) + + val measure = joined.select( + rf_tile_mean(rf_local_subtract($"tile", $"tile2")) as "diff_mean", + rf_tile_stats(rf_local_subtract($"tile", $"tile2")).getField("variance") as "diff_var") + .as[(Double, Double)] + .collect() + all (measure) should be ((0.0, 0.0)) + } + + it("should join same scene in different tile sizes"){ + val r1prime = b4nativeTif.toDF(TileDimensions(25, 25)).withColumnRenamed("tile", "tile2") + r1prime.select(rf_dimensions($"tile2").getField("rows")).as[Int].first() should be (25) + val joined = b4nativeRf.rasterJoin(r1prime) + + joined.count() should be (b4nativeRf.count()) + + val measure = joined.select( + rf_tile_mean(rf_local_subtract($"tile", $"tile2")) as "diff_mean", + rf_tile_stats(rf_local_subtract($"tile", $"tile2")).getField("variance") as "diff_var") + .as[(Double, Double)] + .collect() + all (measure) should be ((0.0, 0.0)) + + } + + it("should join same scene in two projections, same tile size") { + + // b4warpedRf source data is gdal warped b4nativeRf data; join them together. + val joined = b4nativeRf.rasterJoin(b4warpedRf) + // create a Raster from tile2 which should be almost equal to b4nativeTif + val result = joined.agg(TileRasterizerAggregate( + ProjectedRasterDefinition(b4nativeTif.cols, b4nativeTif.rows, b4nativeTif.cellType, b4nativeTif.crs, b4nativeTif.extent, Bilinear), + $"crs", $"extent", $"tile2") as "raster" + ).select(col("raster").as[Raster[Tile]]).first() + + result.extent shouldBe b4nativeTif.extent + + // Test the overall local difference of the `result` versus the original + import geotrellis.raster.mapalgebra.local._ + val sub = b4nativeTif.extent.buffer(-b4nativeTif.extent.width * 0.01) + val diff = Abs( + Subtract( + result.crop(sub).tile.convert(IntConstantNoDataCellType), + b4nativeTif.raster.crop(sub).tile.convert(IntConstantNoDataCellType) + ) + ) + // DN's within arbitrary threshold. N.B. the range of values in the source raster is (6396, 27835) + diff.statisticsDouble.get.mean should be (0.0 +- 200) + // Overall signal is preserved + val b4nativeStddev = b4nativeTif.tile.statisticsDouble.get.stddev + val rel_diff = diff.statisticsDouble.get.mean / b4nativeStddev + rel_diff should be (0.0 +- 0.15) + + // Use the tile structure of the `joined` dataframe to argue that the structure of the image is similar between `b4nativeTif` and `joined.tile2` + val tile_diffs = joined.select((abs(rf_tile_mean($"tile") - rf_tile_mean($"tile2")) / lit( b4nativeStddev)).alias("z")) + + // Check the 90%-ile z score; recognize there will be some localized areas of larger error + tile_diffs.selectExpr("percentile(z, 0.90)").as[Double].first() should be < 0.10 + // Check the median z score; it is pretty close to zero + tile_diffs.selectExpr("percentile(z, 0.50)").as[Double].first() should be < 0.025 + } + + it("should join multiple RHS tile columns"){ + // join multiple native CRS bands to the EPSG 4326 RF + + val multibandRf = b4nativeRf + .withColumn("t_plus", rf_local_add($"tile", $"tile")) + .withColumn("t_mult", rf_local_multiply($"tile", $"tile")) + multibandRf.tileColumns.length should be (3) + + val multibandJoin = multibandRf.rasterJoin(b4warpedRf) + + multibandJoin.tileColumns.length should be (4) + multibandJoin.count() should be (multibandRf.count()) + } + + it("should join with heterogeneous LHS CRS and coverages"){ + + val df17 = readSingleband("m_3607824_se_17_1_20160620_subset.tif") + .toDF(TileDimensions(50, 50)) + .withColumn("utm", lit(17)) + // neighboring and slightly overlapping NAIP scene + val df18 = readSingleband("m_3607717_sw_18_1_20160620_subset.tif") + .toDF(TileDimensions(60, 60)) + .withColumn("utm", lit(18)) + + df17.count() should be (6 * 6) // file is 300 x 300 + df18.count() should be (5 * 5) // file is 300 x 300 + + val df = df17.union(df18) + df.count() should be (6 * 6 + 5 * 5) + val expectCrs = Array("+proj=utm +zone=17 +datum=NAD83 +units=m +no_defs ", "+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ") + df.select($"crs".getField("crsProj4")).distinct().as[String].collect() should contain theSameElementsAs expectCrs + + // read a third source to join. burned in box that intersects both above subsets; but more so on the df17 + val box = readSingleband("m_3607_box.tif").toDF(TileDimensions(4,4)).withColumnRenamed("tile", "burned") + val joined = df.rasterJoin(box) + + joined.count() should be (df.count) + + val totals = joined.groupBy($"utm").agg(sum(rf_tile_sum($"burned")).alias("burned_total")) + val total18 = totals.where($"utm" === 18).select($"burned_total").as[Double].first() + val total17 = totals.where($"utm" === 17).select($"burned_total").as[Double].first() + + total18 should be > 0.0 + total18 should be < total17 + + + } + + it("should pass through ancillary columns") { + val left = b4nativeRf.withColumn("left_id", monotonically_increasing_id()) + val right = b4warpedRf.withColumn("right_id", monotonically_increasing_id()) + val joined = left.rasterJoin(right) + joined.columns should contain allElementsOf Seq("left_id", "right_id_agg") + } + } +} diff --git a/core/src/test/scala/org/locationtech/rasterframes/ReprojectGeometrySpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ReprojectGeometrySpec.scala new file mode 100644 index 000000000..a58294287 --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/ReprojectGeometrySpec.scala @@ -0,0 +1,122 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes + +import geotrellis.proj4.{CRS, LatLng, Sinusoidal, WebMercator} +import org.apache.spark.sql.Encoders +import org.locationtech.jts.geom._ + +/** + * Test for geometry reprojection. + * + * @since 11/29/18 + */ +class ReprojectGeometrySpec extends TestEnvironment { + // Note: Test data copied from ReprojectSpec in GeoTrellis + val fact = new GeometryFactory() + val llLineString: Geometry = fact.createLineString(Array( + new Coordinate(-111.09374999999999, 34.784483415461345), + new Coordinate(-111.09374999999999, 43.29919735147067), + new Coordinate(-75.322265625, 43.29919735147067), + new Coordinate(-75.322265625, 34.784483415461345), + new Coordinate(-111.09374999999999, 34.784483415461345) + )) + + val wmLineString: Geometry = fact.createLineString(Array( + new Coordinate(-12366899.680315234, 4134631.734001753), + new Coordinate(-12366899.680315234, 5357624.186564572), + new Coordinate(-8384836.254770693, 5357624.186564572), + new Coordinate(-8384836.254770693, 4134631.734001753), + new Coordinate(-12366899.680315234, 4134631.734001753) + )) + + describe("Geometry reprojection") { + import spark.implicits._ + + it("should handle two literal CRSs") { + + val df = Seq((llLineString, wmLineString)).toDF("ll", "wm") + + val rp = df.select( + st_reproject($"ll", LatLng, WebMercator) as "wm2", + st_reproject($"wm", WebMercator, LatLng) as "ll2", + st_reproject(st_reproject($"ll", LatLng, Sinusoidal), Sinusoidal, WebMercator) as "wm3" + ).as[(Geometry, Geometry, Geometry)] + + + val (wm2, ll2, wm3) = rp.first() + + wm2 should matchGeom(wmLineString, 0.00001) + ll2 should matchGeom(llLineString, 0.00001) + wm3 should matchGeom(wmLineString, 0.00001) + } + + it("should handle one literal crs") { + implicit val enc = Encoders.tuple(jtsGeometryEncoder, jtsGeometryEncoder, crsEncoder) + val df = Seq((llLineString, wmLineString, LatLng: CRS)).toDF("ll", "wm", "llCRS") + + val rp = df.select( + st_reproject($"ll", $"llCRS", WebMercator) as "wm2", + st_reproject($"wm", WebMercator, $"llCRS") as "ll2", + st_reproject(st_reproject($"ll", $"llCRS", Sinusoidal), Sinusoidal, WebMercator) as "wm3" + ).as[(Geometry, Geometry, Geometry)] + + + val (wm2, ll2, wm3) = rp.first() + + wm2 should matchGeom(wmLineString, 0.00001) + ll2 should matchGeom(llLineString, 0.00001) + wm3 should matchGeom(wmLineString, 0.00001) + } + + it("should accept other geometry types") { + val df = Seq(1, 2, 3).toDF("id") + + noException shouldBe thrownBy { + df.select(st_reproject(st_makePoint($"id", $"id"), WebMercator, Sinusoidal)).count() + } + } + + it("should work in SQL") { + implicit val enc = Encoders.tuple(jtsGeometryEncoder, jtsGeometryEncoder, crsEncoder) + val df = Seq((llLineString, wmLineString, LatLng: CRS)).toDF("ll", "wm", "llCRS") + df.createOrReplaceTempView("geom") + + val rp = spark.sql( + """ + | SELECT st_reproject(ll, llCRS, 'EPSG:3857') as wm2, + | st_reproject(wm, 'EPSG:3857', llCRS) as ll2, + | st_reproject(st_reproject(ll, llCRS, '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs'), + | '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs', 'EPSG:3857') as wm3 + | FROM geom + """.stripMargin).as[(Geometry, Geometry, Geometry)] + + val (wm2, ll2, wm3) = rp.first() + + wm2 should matchGeom(wmLineString, 0.00001) + ll2 should matchGeom(llLineString, 0.00001) + wm3 should matchGeom(wmLineString, 0.00001) + + checkDocs("st_reproject") + } + } +} diff --git a/core/src/test/scala/astraea/spark/rasterframes/SpatialKeySpec.scala b/core/src/test/scala/org/locationtech/rasterframes/SpatialKeySpec.scala similarity index 87% rename from core/src/test/scala/astraea/spark/rasterframes/SpatialKeySpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/SpatialKeySpec.scala index 065e9a5ed..b99b5c48e 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/SpatialKeySpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/SpatialKeySpec.scala @@ -15,11 +15,12 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes -import com.vividsolutions.jts.geom.Polygon import geotrellis.proj4.LatLng import geotrellis.vector.Point import org.locationtech.geomesa.curve.Z2SFC @@ -37,11 +38,11 @@ class SpatialKeySpec extends TestEnvironment with TestData { describe("Spatial key conversions") { val raster = sampleGeoTiff.projectedRaster // Create a raster frame with a single row - val rf = raster.toRF(raster.tile.cols, raster.tile.rows) + val rf = raster.toLayer(raster.tile.cols, raster.tile.rows) it("should add an extent column") { val expected = raster.extent.jtsGeom - val result = rf.withBounds().select($"bounds".as[Polygon]).first + val result = rf.withGeometry().select(GEOMETRY_COLUMN).first assert(result === expected) } @@ -64,6 +65,4 @@ class SpatialKeySpec extends TestEnvironment with TestData { assert(result === expected) } } - // This is to avoid an IntelliJ error - protected def withFixture(test: Any) = ??? } diff --git a/core/src/test/scala/astraea/spark/rasterframes/TestData.scala b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala similarity index 76% rename from core/src/test/scala/astraea/spark/rasterframes/TestData.scala rename to core/src/test/scala/org/locationtech/rasterframes/TestData.scala index 29c06849e..1b6b373e9 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/TestData.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright (c) 2017. Astraea, Inc. + * Copyright 2017 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,19 +14,17 @@ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes import java.net.URI -import java.nio.file.Paths +import java.nio.file.{Files, Paths} import java.time.ZonedDateTime -import astraea.spark.rasterframes.expressions.tilestats.NoDataCells -import astraea.spark.rasterframes.model.TileContext -import astraea.spark.rasterframes.tiles.ProjectedRasterTile -import astraea.spark.rasterframes.{functions => F} -import com.vividsolutions.jts.geom.{Coordinate, GeometryFactory} import geotrellis.proj4.{CRS, LatLng} import geotrellis.raster import geotrellis.raster._ @@ -38,6 +36,10 @@ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.commons.io.IOUtils import org.apache.spark.SparkContext import org.apache.spark.sql.SparkSession +import org.locationtech.jts.geom.{Coordinate, GeometryFactory} +import org.locationtech.rasterframes.expressions.tilestats.NoDataCells +import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import spray.json.JsObject import scala.reflect.ClassTag @@ -109,6 +111,12 @@ trait TestData { require((1 to 11).contains(band), "Invalid band number") readSingleband(s"L8-B$band-Elkton-VA.tiff") } + + def l8SamplePath(band: Int) = { + require((1 to 11).contains(band), "Invalid band number") + getClass.getResource(s"/L8-B$band-Elkton-VA.tiff").toURI + } + def l8Labels = readSingleband("L8-Labels-Elkton-VA.tiff") def naipSample(band: Int) = { @@ -116,22 +124,36 @@ trait TestData { readSingleband(s"NAIP-VA-b$band.tiff") } - def rgbCogSample = readMultiband("LC08_RGB_Norfolk_COG.tiff") + def rgbCogSample = readMultiband("LC08_RGB_Norfolk_COG.tiff") + + def rgbCogSamplePath = getClass.getResource("/LC08_RGB_Norfolk_COG.tiff").toURI def sampleTileLayerRDD(implicit spark: SparkSession): TileLayerRDD[SpatialKey] = { - val rf = sampleGeoTiff.projectedRaster.toRF(128, 128) + val rf = sampleGeoTiff.projectedRaster.toLayer(128, 128) rf.toTileLayerRDD(rf.tileColumns.head).left.get } private val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_%s.TIF" - lazy val remoteCOGSingleband1 = URI.create(baseCOG.format("B1")) - lazy val remoteCOGSingleband2 = URI.create(baseCOG.format("B2")) + lazy val remoteCOGSingleband1: URI = URI.create(baseCOG.format("B1")) + lazy val remoteCOGSingleband2: URI = URI.create(baseCOG.format("B2")) + + lazy val remoteCOGMultiband: URI = URI.create("https://s3-us-west-2.amazonaws.com/radiant-nasa-iserv/2014/02/14/IP0201402141023382027S03100E/IP0201402141023382027S03100E-COG.tif") + + lazy val remoteMODIS: URI = URI.create("https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF") + lazy val remoteL8: URI = URI.create("https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/017/033/LC08_L1TP_017033_20181010_20181030_01_T1/LC08_L1TP_017033_20181010_20181030_01_T1_B4.TIF") + lazy val remoteHttpMrfPath: URI = URI.create("https://s3.amazonaws.com/s22s-rasterframes-integration-tests/m_3607526_sw_18_1_20160708.mrf") + lazy val remoteS3MrfPath: URI = URI.create("s3://naip-analytic/va/2016/100cm/rgbir/37077/m_3707764_sw_18_1_20160708.mrf") - lazy val remoteCOGMultiband = URI.create("https://s3-us-west-2.amazonaws.com/radiant-nasa-iserv/2014/02/14/IP0201402141023382027S03100E/IP0201402141023382027S03100E-COG.tif") + lazy val localSentinel: URI = getClass.getResource("/B01.jp2").toURI + lazy val cogPath: URI = getClass.getResource("/LC08_RGB_Norfolk_COG.tiff").toURI + lazy val singlebandCogPath: URI = getClass.getResource("/LC08_B7_Memphis_COG.tiff").toURI + lazy val nonCogPath: URI = getClass.getResource("/L8-B8-Robinson-IL.tiff").toURI - lazy val remoteMODIS = URI.create("https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF") + lazy val l8B1SamplePath: URI = l8SamplePath(1) + lazy val l8samplePath: URI = getClass.getResource("/L8-B1-Elkton-VA.tiff").toURI + lazy val modisConvertedMrfPath: URI = getClass.getResource("/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf").toURI - object JTS { + object GeomData { val fact = new GeometryFactory() val c1 = new Coordinate(1, 2) val c2 = new Coordinate(3, 4) @@ -144,6 +166,19 @@ trait TestData { val mpoly = fact.createMultiPolygon(Array(poly, poly, poly)) val coll = fact.createGeometryCollection(Array(point, line, poly, mpoint, mline, mpoly)) val all = Seq(point, line, poly, mpoint, mline, mpoly, coll) + lazy val geoJson = { + import scala.collection.JavaConversions._ + val p = Paths.get(TestData.getClass + .getResource("/L8-Labels-Elkton-VA.geojson").toURI) + Files.readAllLines(p).mkString("\n") + } + lazy val features = { + import geotrellis.vector.io._ + import geotrellis.vector.io.json.JsonFeatureCollection + import spray.json.DefaultJsonProtocol._ + import spray.json._ + GeomData.geoJson.parseGeoJson[JsonFeatureCollection].getAllPolygonFeatures[JsObject]() + } } } diff --git a/core/src/test/scala/astraea/spark/rasterframes/TestEnvironment.scala b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala similarity index 74% rename from core/src/test/scala/astraea/spark/rasterframes/TestEnvironment.scala rename to core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala index aaf173014..87ab2559d 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/TestEnvironment.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright (c) 2017. Astraea, Inc. + * Copyright 2017 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,25 +14,25 @@ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes import java.nio.file.{Files, Paths} -import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.stringEnc -import astraea.spark.rasterframes.ref.RasterSource -import astraea.spark.rasterframes.ref.RasterSource.ReadCallback -import astraea.spark.rasterframes.util.toParquetFriendlyColumnName -import com.vividsolutions.jts.geom.Geometry +import com.typesafe.scalalogging.LazyLogging import geotrellis.spark.testkit.{TestEnvironment => GeoTrellisTestEnvironment} -import geotrellis.util.LazyLogging import org.apache.spark.SparkContext import org.apache.spark.sql._ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructType +import org.locationtech.jts.geom.Geometry import org.scalactic.Tolerance import org.scalatest._ import org.scalatest.matchers.{MatchResult, Matcher} +import org.locationtech.rasterframes.util._ trait TestEnvironment extends FunSpec with GeoTrellisTestEnvironment with Matchers with Inspectors with Tolerance with LazyLogging { @@ -40,11 +40,14 @@ trait TestEnvironment extends FunSpec with GeoTrellisTestEnvironment override def sparkMaster: String = "local[*]" override implicit def sc: SparkContext = { _sc.setLogLevel("ERROR"); _sc } - //p.setProperty(“spark.driver.allowMultipleContexts”, “true”) lazy val sqlContext: SQLContext = { - val session = SparkSession.builder.config(_sc.getConf).getOrCreate() - astraea.spark.rasterframes.WithSQLContextMethods(session.sqlContext).withRasterFrames + val session = SparkSession.builder + .config(_sc.getConf) + .config("spark.sql.crossJoin.enabled", true) + .withKryoSerialization + .getOrCreate() + session.sqlContext.withRasterFrames } lazy val sql: String ⇒ DataFrame = sqlContext.sql @@ -86,6 +89,7 @@ trait TestEnvironment extends FunSpec with GeoTrellisTestEnvironment def matchGeom(g: Geometry, tolerance: Double) = new GeometryMatcher(g, tolerance) def checkDocs(name: String): Unit = { + import spark.implicits._ val docs = sql(s"DESCRIBE FUNCTION EXTENDED $name").as[String].collect().mkString("\n") docs should include(name) docs shouldNot include("not found") @@ -95,16 +99,5 @@ trait TestEnvironment extends FunSpec with GeoTrellisTestEnvironment } object TestEnvironment { - case class ReadMonitor(ignoreHeader: Boolean = true) extends ReadCallback with LazyLogging { - var reads: Int = 0 - var total: Long = 0 - override def readRange(source: RasterSource, start: Long, length: Int): Unit = { - logger.trace(s"Reading $length at $start from $source") - // Ignore header reads - if(!ignoreHeader || start > 0) reads += 1 - total += length - } - override def toString: String = s"$productPrefix(reads=$reads, total=$total)" - } } \ No newline at end of file diff --git a/core/src/test/scala/astraea/spark/rasterframes/TileAssemblerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/TileAssemblerSpec.scala similarity index 78% rename from core/src/test/scala/astraea/spark/rasterframes/TileAssemblerSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/TileAssemblerSpec.scala index 29eff421f..757231595 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/TileAssemblerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TileAssemblerSpec.scala @@ -19,16 +19,15 @@ * */ -package astraea.spark.rasterframes -import astraea.spark.rasterframes.ref.RasterSource -import astraea.spark.rasterframes.ref.RasterSource.InMemoryRasterSource +package org.locationtech.rasterframes + import com.typesafe.scalalogging.LazyLogging import geotrellis.proj4.LatLng import geotrellis.raster._ import geotrellis.raster.render.ColorRamps import geotrellis.vector.Extent -import org.apache.spark.sql._ -import org.apache.spark.sql.{functions => F} +import org.apache.spark.sql.{functions => F, _} +import org.locationtech.rasterframes.ref.{InMemoryRasterSource, RasterSource} /** * @@ -42,15 +41,15 @@ class TileAssemblerSpec extends TestEnvironment { it("should reassemble a small scene") { val raster = TestData.l8Sample(8).projectedRaster - val rf = raster.toRF(16, 16) + val rf = raster.toLayer(16, 16) val ct = rf.tileLayerMetadata.merge.cellType val (tileCols, tileRows) = rf.tileLayerMetadata.merge.tileLayout.tileDimensions - val exploded = rf.select($"spatial_key", explode_tiles($"tile")) + val exploded = rf.select($"spatial_key", rf_explode_tiles($"tile")) val assembled = exploded .groupBy($"spatial_key") - .agg(assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, $"tile", tileCols, tileRows, ct)) + .agg(rf_assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, $"tile", tileCols, tileRows, ct)) assert( @@ -65,12 +64,13 @@ class TileAssemblerSpec extends TestEnvironment { val sceneSize = (260, 257) val rs = InMemoryRasterSource(TestData.randomTile(sceneSize._1, sceneSize._2, ByteConstantNoDataCellType), Extent(10, 20, 30, 40), LatLng) val df = rs.toDF - val exploded = df.select($"spatial_index", $"extent", tile_dimensions($"tile") as "tile_dimensions", explode_tiles($"tile")) + val exploded = df.select($"spatial_index", $"extent", rf_dimensions($"tile") as "tile_dimensions", rf_explode_tiles($"tile")) val assembled = exploded .groupBy($"spatial_index", $"extent", $"tile_dimensions") .agg( - convert_cell_type(assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, + rf_convert_cell_type( + rf_assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, $"tile", $"tile_dimensions.cols", $"tile_dimensions.rows"), rs.cellType) as "tile" ) @@ -89,7 +89,7 @@ class TileAssemblerSpec extends TestEnvironment { val exploded = util.time("exploded") { df - .select($"spatial_index", explode_tiles($"tile")) + .select($"spatial_index", rf_explode_tiles($"tile")) .forceCache } @@ -98,7 +98,7 @@ class TileAssemblerSpec extends TestEnvironment { val assembled = util.time("assembled") { exploded .groupBy($"spatial_index") - .agg(assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, + .agg(rf_assemble_tile(COLUMN_INDEX_COLUMN, ROW_INDEX_COLUMN, $"tile", 256, 256, UShortUserDefinedNoDataCellType(32767))) .forceCache @@ -111,8 +111,8 @@ class TileAssemblerSpec extends TestEnvironment { assert(assembled.count() === df.count()) - val expected = df.select(agg_stats($"tile")).first() - val result = assembled.select(agg_stats($"tile")).first() + val expected = df.select(rf_agg_stats($"tile")).first() + val result = assembled.select(rf_agg_stats($"tile")).first() assert(result.copy(no_data_cells = expected.no_data_cells) === expected) } @@ -134,9 +134,9 @@ object TileAssemblerSpec extends LazyLogging { implicit class WithToDF(val rs: RasterSource) { def toDF(implicit spark: SparkSession): DataFrame = { import spark.implicits._ - rs.readAll().left.get + rs.readAll() .zipWithIndex - .map { case (r, i) ⇒ (i, r.extent, r.tile) } + .map { case (r, i) ⇒ (i, r.extent, r.tile.band(0)) } .toDF("spatial_index", "extent", "tile") .repartition($"spatial_index") .forceCache diff --git a/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/TileStatsSpec.scala similarity index 78% rename from core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/TileStatsSpec.scala index 781b8290d..50920ab1c 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TileStatsSpec.scala @@ -15,18 +15,18 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes -import astraea.spark.rasterframes.TestData.randomTile -import astraea.spark.rasterframes.TestData.fracTile -import astraea.spark.rasterframes.expressions.aggstats.LocalMeanAggregate -import astraea.spark.rasterframes.stats.CellHistogram import geotrellis.raster._ -import geotrellis.spark._ import geotrellis.raster.mapalgebra.local.{Max, Min} +import geotrellis.spark._ import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes.TestData.randomTile +import org.locationtech.rasterframes.stats.CellHistogram /** * Test rig associated with computing statistics and other descriptive @@ -35,21 +35,21 @@ import org.apache.spark.sql.functions._ * @since 9/18/17 */ class TileStatsSpec extends TestEnvironment with TestData { - import sqlContext.implicits._ import TestData.injectND + import sqlContext.implicits._ describe("computing statistics over tiles") { //import org.apache.spark.sql.execution.debug._ it("should report dimensions") { val df = Seq[(Tile, Tile)]((byteArrayTile, byteArrayTile)).toDF("tile1", "tile2") - val dims = df.select(tile_dimensions($"tile1") as "dims").select("dims.*") + val dims = df.select(rf_dimensions($"tile1") as "dims").select("dims.*") assert(dims.as[(Int, Int)].first() === (3, 3)) assert(dims.schema.head.name === "cols") val query = sql("""|select dims.* from ( - |select rf_tile_dimensions(tiles) as dims from ( + |select rf_dimensions(tiles) as dims from ( |select rf_make_constant_tile(1, 10, 10, 'int8raw') as tiles)) |""".stripMargin) write(query) @@ -57,7 +57,7 @@ class TileStatsSpec extends TestEnvironment with TestData { df.repartition(4).createOrReplaceTempView("tmp") assert( - sql("select dims.* from (select rf_tile_dimensions(tile2) as dims from tmp)") + sql("select dims.* from (select rf_dimensions(tile2) as dims from tmp)") .as[(Int, Int)] .first() === (3, 3)) } @@ -67,20 +67,20 @@ class TileStatsSpec extends TestEnvironment with TestData { forEvery(ct) { c => val expected = CellType.fromName(c) val tile = randomTile(5, 5, expected) - val result = Seq(tile).toDF("tile").select(cell_type($"tile")).first() + val result = Seq(tile).toDF("tile").select(rf_cell_type($"tile")).first() result should be(expected) } } // tiles defined for the next few tests - val tile1 = fracTile(10, 10, 5) + val tile1 = TestData.fracTile(10, 10, 5) val tile2 = ArrayTile(Array(-5, -4, -3, -2, -1, 0, 1, 2, 3), 3, 3) val tile3 = randomTile(255, 255, IntCellType) it("should compute accurate item counts") { val ds = Seq[Tile](tile1, tile2, tile3).toDF("tiles") val checkedValues = Seq[Double](0, 4, 7, 13, 26) - val result = checkedValues.map(x => ds.select(tile_histogram($"tiles")).first().itemCount(x)) + val result = checkedValues.map(x => ds.select(rf_tile_histogram($"tiles")).first().itemCount(x)) forEvery(checkedValues) { x => assert((x == 0 && result.head == 4) || result.contains(x - 1)) } @@ -89,7 +89,7 @@ class TileStatsSpec extends TestEnvironment with TestData { it("Should compute quantiles") { val ds = Seq[Tile](tile1, tile2, tile3).toDF("tiles") val numBreaks = 5 - val breaks = ds.select(tile_histogram($"tiles")).map(_.quantileBreaks(numBreaks)).collect() + val breaks = ds.select(rf_tile_histogram($"tiles")).map(_.quantileBreaks(numBreaks)).collect() assert(breaks(1).length === numBreaks) assert(breaks(0).apply(2) == 25) assert(breaks(1).max <= 3 && breaks.apply(1).min >= -5) @@ -101,7 +101,7 @@ class TileStatsSpec extends TestEnvironment with TestData { ds.createOrReplaceTempView("tmp") withClue("max") { - val max = ds.agg(agg_local_max($"tiles")) + val max = ds.agg(rf_agg_local_max($"tiles")) val expected = Max(byteArrayTile, byteConstantTile) write(max) assert(max.as[Tile].first() === expected) @@ -112,7 +112,7 @@ class TileStatsSpec extends TestEnvironment with TestData { } withClue("min") { - val min = ds.agg(agg_local_min($"tiles")) + val min = ds.agg(rf_agg_local_min($"tiles")) val expected = Min(byteArrayTile, byteConstantTile) write(min) assert(min.as[Tile].first() === Min(byteArrayTile, byteConstantTile)) @@ -127,19 +127,19 @@ class TileStatsSpec extends TestEnvironment with TestData { withClue("mean") { val ds = Seq.fill[Tile](3)(randomTile(5, 5, FloatConstantNoDataCellType)).toDS() - val means1 = ds.select(tile_stats($"value")).map(_.mean).collect - val means2 = ds.select(tile_mean($"value")).collect + val means1 = ds.select(rf_tile_stats($"value")).map(_.mean).collect + val means2 = ds.select(rf_tile_mean($"value")).collect // Compute the mean manually, knowing we're not dealing with no-data values. val means = - ds.select(tile_to_array_double($"value")).map(a => a.sum / a.length).collect + ds.select(rf_tile_to_array_double($"value")).map(a => a.sum / a.length).collect forAll(means.zip(means1)) { case (l, r) => assert(l === r +- 1e-6) } forAll(means.zip(means2)) { case (l, r) => assert(l === r +- 1e-6) } } withClue("sum") { - val rf = l8Sample(1).projectedRaster.toRF + val rf = l8Sample(1).toDF() val expected = 309149454 // computed with rasterio - val result = rf.agg(sum(tile_sum($"tile"))).collect().head.getDouble(0) + val result = rf.agg(sum(rf_tile_sum($"tile"))).collect().head.getDouble(0) logger.info(s"L8 sample band 1 grand total: ${result}") assert(result === expected) } @@ -149,7 +149,7 @@ class TileStatsSpec extends TestEnvironment with TestData { val ds = Seq.fill[Tile](3)(randomTile(5, 5, FloatCellType)).toDF("tiles") ds.createOrReplaceTempView("tmp") - val r1 = ds.select(tile_histogram($"tiles")) + val r1 = ds.select(rf_tile_histogram($"tiles")) assert(r1.first.totalCount === 5 * 5) write(r1) val r2 = sql("select hist.* from (select rf_tile_histogram(tiles) as hist from tmp)").as[CellHistogram] @@ -179,7 +179,7 @@ class TileStatsSpec extends TestEnvironment with TestData { .fill[Tile](rows)(randomTile(tileSize, tileSize, FloatConstantNoDataCellType)) .toDF("tiles") ds.createOrReplaceTempView("tmp") - val agg = ds.select(agg_approx_histogram($"tiles")) + val agg = ds.select(rf_agg_approx_histogram($"tiles")) val histArray = agg.collect() histArray.length should be (1) @@ -198,21 +198,21 @@ class TileStatsSpec extends TestEnvironment with TestData { it("should compute aggregate mean") { val ds = (Seq.fill[Tile](10)(randomTile(5, 5, FloatCellType)) :+ null).toDF("tiles") - val agg = ds.select(agg_mean($"tiles")) - val stats = ds.select(agg_stats($"tiles") as "stats").select($"stats.mean".as[Double]) + val agg = ds.select(rf_agg_mean($"tiles")) + val stats = ds.select(rf_agg_stats($"tiles") as "stats").select($"stats.mean".as[Double]) assert(agg.first() === stats.first()) } it("should compute aggregate statistics") { val ds = Seq.fill[Tile](10)(randomTile(5, 5, FloatConstantNoDataCellType)).toDF("tiles") - val exploded = ds.select(explode_tiles($"tiles")) + val exploded = ds.select(rf_explode_tiles($"tiles")) val (mean, vrnc) = exploded.agg(avg($"tiles"), var_pop($"tiles")).as[(Double, Double)].first - val stats = ds.select(agg_stats($"tiles") as "stats") ///.as[(Long, Double, Double, Double, Double)] + val stats = ds.select(rf_agg_stats($"tiles") as "stats") ///.as[(Long, Double, Double, Double, Double)] //stats.printSchema() noException shouldBe thrownBy { - ds.select(agg_stats($"tiles")).collect() + ds.select(rf_agg_stats($"tiles")).collect() } val agg = stats.select($"stats.variance".as[Double]) @@ -223,7 +223,7 @@ class TileStatsSpec extends TestEnvironment with TestData { val agg2 = sql("select stats.* from (select rf_agg_stats(tiles) as stats from tmp)") assert(agg2.first().getAs[Long]("data_cells") === 250L) - val agg3 = ds.agg(agg_stats($"tiles") as "stats").select($"stats.mean".as[Double]) + val agg3 = ds.agg(rf_agg_stats($"tiles") as "stats").select($"stats.mean".as[Double]) assert(mean === agg3.first()) } @@ -236,7 +236,7 @@ class TileStatsSpec extends TestEnvironment with TestData { .map(injectND(2)) :+ null).toDF("tiles") ds.createOrReplaceTempView("tmp") - val agg = ds.select(agg_local_stats($"tiles") as "stats") + val agg = ds.select(rf_agg_local_stats($"tiles") as "stats") val stats = agg.select("stats.*") //printStatsRows(stats) @@ -269,25 +269,25 @@ class TileStatsSpec extends TestEnvironment with TestData { val dsNd = (Seq.fill(20)(completeTile) :+ incompleteTile :+ null).toDF("tiles") // counted everything properly - val countTile = ds.select(agg_local_data_cells($"tiles")).first() + val countTile = ds.select(rf_agg_local_data_cells($"tiles")).first() forAll(countTile.toArray())(i => assert(i === 20)) - val countArray = dsNd.select(agg_local_data_cells($"tiles")).first().toArray() + val countArray = dsNd.select(rf_agg_local_data_cells($"tiles")).first().toArray() val expectedCount = (completeTile.localDefined().toArray zip incompleteTile.localDefined().toArray()).toSeq.map( pr => pr._1 * 20 + pr._2) assert(countArray === expectedCount) - val countNodataArray = dsNd.select(agg_local_no_data_cells($"tiles")).first().toArray + val countNodataArray = dsNd.select(rf_agg_local_no_data_cells($"tiles")).first().toArray assert(countNodataArray === incompleteTile.localUndefined().toArray) - val minTile = dsNd.select(agg_local_min($"tiles")).first() + val minTile = dsNd.select(rf_agg_local_min($"tiles")).first() assert(minTile.toArray() === completeTile.toArray()) - val maxTile = dsNd.select(agg_local_max($"tiles")).first() + val maxTile = dsNd.select(rf_agg_local_max($"tiles")).first() assert(maxTile.toArray() === completeTile.toArray()) - val meanTile = dsNd.select(agg_local_mean($"tiles")).first() + val meanTile = dsNd.select(rf_agg_local_mean($"tiles")).first() assert(meanTile.toArray() === completeTile.toArray()) } } @@ -300,20 +300,20 @@ class TileStatsSpec extends TestEnvironment with TestData { .map(injectND(nds)) :+ null).toDF("tiles") it("should count cells by NoData state") { - val counts = tiles.select(no_data_cells($"tiles")).collect().dropRight(1) + val counts = tiles.select(rf_no_data_cells($"tiles")).collect().dropRight(1) forEvery(counts)(c => assert(c === nds)) - val counts2 = tiles.select(data_cells($"tiles")).collect().dropRight(1) + val counts2 = tiles.select(rf_data_cells($"tiles")).collect().dropRight(1) forEvery(counts2)(c => assert(c === tsize * tsize - nds)) } it("should detect all NoData tiles") { - val ndCount = tiles.select("*").where(is_no_data_tile($"tiles")).count() + val ndCount = tiles.select("*").where(rf_is_no_data_tile($"tiles")).count() ndCount should be(1) val ndTiles = (Seq.fill[Tile](count)(ArrayTile.empty(UByteConstantNoDataCellType, tsize, tsize)) :+ null) .toDF("tiles") - val ndCount2 = ndTiles.select("*").where(is_no_data_tile($"tiles")).count() + val ndCount2 = ndTiles.select("*").where(rf_is_no_data_tile($"tiles")).count() ndCount2 should be(count + 1) } } diff --git a/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala similarity index 80% rename from core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala index b83b94486..3081b2f64 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala @@ -1,5 +1,3 @@ - - /* * This software is licensed under the Apache 2 license, quoted below. * @@ -17,19 +15,21 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes - -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.functions.cellTypes +package org.locationtech.rasterframes import geotrellis.raster.{CellType, Tile} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf._ +import org.apache.spark.sql.types.StringType +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.tiles.ShowableTile import org.scalatest.Inspectors /** - * RasterFrame test rig. + * RasterFrameLayer test rig. * * @since 7/10/17 */ @@ -38,12 +38,11 @@ class TileUDTSpec extends TestEnvironment with TestData with Inspectors { spark.version val tileEncoder: ExpressionEncoder[Tile] = ExpressionEncoder() - val TileType = new TileUDT() implicit val ser = TileUDT.tileSerializer describe("TileUDT") { - val tileSizes = Seq(2, 64, 128, 222, 511) - val ct = cellTypes().filter(_ != "bool") + val tileSizes = Seq(2, 7, 64, 128, 511) + val ct = functions.cellTypes().filter(_ != "bool") def forEveryConfig(test: Tile ⇒ Unit): Unit = { forEvery(tileSizes.combinations(2).toSeq) { case Seq(cols, rows) ⇒ @@ -93,5 +92,13 @@ class TileUDTSpec extends TestEnvironment with TestData with Inspectors { } } } + + it("should provide a pretty-print tile") { + import spark.implicits._ + forEveryConfig { tile => + val stringified = Seq(tile).toDF("tile").select($"tile".cast(StringType)).as[String].first() + stringified should be(ShowableTile.show(tile)) + } + } } } diff --git a/core/src/test/scala/astraea/spark/rasterframes/encoders/CatalystSerializerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala similarity index 86% rename from core/src/test/scala/astraea/spark/rasterframes/encoders/CatalystSerializerSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala index c489b8d7b..4e8bfdfcc 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/encoders/CatalystSerializerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala @@ -19,19 +19,20 @@ * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders + import java.time.ZonedDateTime -import astraea.spark.rasterframes.encoders.StandardEncoders._ -import astraea.spark.rasterframes.model.{CellContext, TileContext, TileDataContext, TileDimensions} -import astraea.spark.rasterframes.ref.{RasterRef, RasterSource} -import astraea.spark.rasterframes.{TestData, TestEnvironment} import geotrellis.proj4._ import geotrellis.raster.{CellSize, CellType, TileLayout, UShortUserDefinedNoDataCellType} import geotrellis.spark.tiling.LayoutDefinition -import geotrellis.spark.{Bounds, KeyBounds, SpaceTimeKey, SpatialKey, TileLayerMetadata} +import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TileLayerMetadata} import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.locationtech.rasterframes.{TestData, TestEnvironment} +import org.locationtech.rasterframes.encoders.StandardEncoders._ +import org.locationtech.rasterframes.model.{CellContext, TileContext, TileDataContext, TileDimensions} +import org.locationtech.rasterframes.ref.{RasterRef, RasterSource} import org.scalatest.Assertion class CatalystSerializerSpec extends TestEnvironment with TestData { @@ -102,8 +103,9 @@ class CatalystSerializerSpec extends TestEnvironment with TestData { } it("should serialize RasterRef") { + // TODO: Decide if RasterRef should be encoded 'flat', non-'flat', or depends val src = RasterSource(remoteCOGSingleband1) - val value = RasterRef(src, Some(src.extent.buffer(-3.0))) + val value = RasterRef(src, 0, Some(src.extent.buffer(-3.0))) assertConsistent(value) assertInvertable(value) } @@ -116,17 +118,17 @@ class CatalystSerializerSpec extends TestEnvironment with TestData { assertContract(ext) } - it("should eserialize ProjectedExtent") { + it("should serialize ProjectedExtent") { val pe = ProjectedExtent(ext, ConusAlbers) assertContract(pe) } - it("should eserialize SpatialKey") { + it("should serialize SpatialKey") { val v = SpatialKey(2, 3) assertContract(v) } - it("should eserialize SpaceTimeKey") { + it("should serialize SpaceTimeKey") { val v = SpaceTimeKey(2, 3, ZonedDateTime.now()) assertContract(v) } diff --git a/core/src/test/scala/astraea/spark/rasterframes/encoders/EncodingSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala similarity index 90% rename from core/src/test/scala/astraea/spark/rasterframes/encoders/EncodingSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala index a0c0bad0e..b27d5cccd 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/encoders/EncodingSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala @@ -19,14 +19,13 @@ * */ -package astraea.spark.rasterframes.encoders +package org.locationtech.rasterframes.encoders import java.io.File import java.net.URI -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.tiles.ProjectedRasterTile -import com.vividsolutions.jts.geom.Envelope +import org.locationtech.rasterframes._ +import org.locationtech.jts.geom.Envelope import geotrellis.proj4._ import geotrellis.raster.{CellType, Tile, TileFeature} import geotrellis.spark.{SpaceTimeKey, SpatialKey, TemporalProjectedExtent, TileLayerMetadata} @@ -34,6 +33,8 @@ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.Row import org.apache.spark.sql.functions._ import org.apache.spark.sql.rf.TileUDT +import org.locationtech.rasterframes.TestEnvironment +import org.locationtech.rasterframes.tiles.ProjectedRasterTile /** * Test rig for encoding GT types into Catalyst types. @@ -151,5 +152,13 @@ class EncodingSpec extends TestEnvironment with TestData { assert(ds.first === env) } } + describe("Dataframe encoding ops on spatial types") { + it("should code RDD[Point]") { + val points = Seq(null, extent.center.jtsGeom, null) + val ds = points.toDS + write(ds) + assert(ds.collect().toSeq === points) + } + } } diff --git a/core/src/test/scala/org/locationtech/rasterframes/expressions/ProjectedLayerMetadataAggregateTest.scala b/core/src/test/scala/org/locationtech/rasterframes/expressions/ProjectedLayerMetadataAggregateTest.scala new file mode 100644 index 000000000..4d4949357 --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/expressions/ProjectedLayerMetadataAggregateTest.scala @@ -0,0 +1,59 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions + +import geotrellis.raster.Tile +import geotrellis.spark._ +import geotrellis.spark.tiling.FloatingLayoutScheme +import geotrellis.vector.{Extent, ProjectedExtent} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.serialized_literal +import org.locationtech.rasterframes.expressions.aggregates.ProjectedLayerMetadataAggregate +import org.locationtech.rasterframes.model.TileDimensions + +class ProjectedLayerMetadataAggregateTest extends TestEnvironment { + + import spark.implicits._ + + describe("ProjectedLayerMetadataAggregate") { + it("should collect metadata from RasterFrame") { + val image = TestData.sampleGeoTiff + val rf = image.projectedRaster.toLayer(60, 65) + val crs = rf.crs + + val df = rf.withExtent() + .select($"extent", $"tile").as[(Extent, Tile)] + + val tileDims = rf.tileLayerMetadata.merge.tileLayout.tileDimensions + + val (_, tlm) = df + .map { case (ext, tile) => (ProjectedExtent(ext, crs), tile) } + .rdd.collectMetadata[SpatialKey](FloatingLayoutScheme(tileDims._1, tileDims._2)) + + val md = df.select(ProjectedLayerMetadataAggregate(crs, TileDimensions(tileDims), $"extent", + serialized_literal(crs), rf_cell_type($"tile"), rf_dimensions($"tile"))) + val tlm2 = md.first() + + tlm2 should be(tlm) + } + } +} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ml/NoDataFilterSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ml/NoDataFilterSpec.scala similarity index 88% rename from core/src/test/scala/astraea/spark/rasterframes/ml/NoDataFilterSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/ml/NoDataFilterSpec.scala index 17a0f25d4..1d4dbc4f6 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/ml/NoDataFilterSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ml/NoDataFilterSpec.scala @@ -15,13 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.ml +package org.locationtech.rasterframes.ml import java.nio.file.Files -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.TestEnvironment import org.scalatest.BeforeAndAfter /** diff --git a/core/src/test/scala/org/locationtech/rasterframes/ml/TileExploderSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ml/TileExploderSpec.scala new file mode 100644 index 000000000..2d9e2d04c --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/ml/TileExploderSpec.scala @@ -0,0 +1,48 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ml + +import org.locationtech.rasterframes.TestData +import geotrellis.raster.Tile +import org.apache.spark.sql.functions.lit +import org.locationtech.rasterframes.TestEnvironment +/** + * + * @since 2/16/18 + */ +class TileExploderSpec extends TestEnvironment with TestData { + describe("Tile explode transformer") { + it("should explode tiles") { + import spark.implicits._ + val df = Seq[(Tile, Tile)]((byteArrayTile, byteArrayTile)).toDF("tile1", "tile2").withColumn("other", lit("stuff")) + + val exploder = new TileExploder() + val newSchema = exploder.transformSchema(df.schema) + + val exploded = exploder.transform(df) + assert(newSchema === exploded.schema) + assert(exploded.columns.length === 5) + assert(exploded.count() === 9) + write(exploded) + } + } +} diff --git a/core/src/test/scala/org/locationtech/rasterframes/model/LazyCRSSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/model/LazyCRSSpec.scala new file mode 100644 index 000000000..1762c402e --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/model/LazyCRSSpec.scala @@ -0,0 +1,43 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.model + +import geotrellis.proj4.{CRS, LatLng, Sinusoidal, WebMercator} +import org.scalatest._ + +class LazyCRSSpec extends FunSpec with Matchers { + val sinPrj = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs" + val llPrj = "epsg:4326" + describe("LazyCRS") { + it("should implement equals") { + LazyCRS(WebMercator) should be(LazyCRS(WebMercator)) + LazyCRS(WebMercator) should be(WebMercator) + WebMercator should be(LazyCRS(WebMercator)) + LazyCRS(sinPrj) should be (Sinusoidal) + CRS.fromString(sinPrj) should be (LazyCRS(Sinusoidal)) + LazyCRS(llPrj) should be(LatLng) + LazyCRS(LatLng) should be(LatLng) + LatLng should be(LazyCRS(llPrj)) + LatLng should be(LazyCRS(LatLng)) + } + } +} diff --git a/core/src/test/scala/astraea/spark/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala similarity index 69% rename from core/src/test/scala/astraea/spark/rasterframes/ref/RasterRefSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala index 4efe2b474..f34c89859 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/ref/RasterRefSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala @@ -19,15 +19,16 @@ * */ -package astraea.spark.rasterframes.ref +package org.locationtech.rasterframes.ref -import astraea.spark.rasterframes.TestEnvironment.ReadMonitor -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.expressions.transformers._ -import astraea.spark.rasterframes.expressions.accessors._ -import astraea.spark.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.expressions.accessors._ +import org.locationtech.rasterframes.expressions.generators._ +import RasterRef.RasterRefTile import geotrellis.raster.Tile import geotrellis.vector.Extent +import org.apache.spark.sql.Encoders +import org.locationtech.rasterframes.TestEnvironment /** * @@ -36,6 +37,7 @@ import geotrellis.vector.Extent */ //noinspection TypeAnnotation class RasterRefSpec extends TestEnvironment with TestData { + def sub(e: Extent) = { val c = e.center val w = e.width @@ -44,15 +46,15 @@ class RasterRefSpec extends TestEnvironment with TestData { } trait Fixture { - val counter = new ReadMonitor - val src = RasterSource(remoteCOGSingleband1, Some(counter)) - val fullRaster = RasterRef(src) + val src = RasterSource(remoteCOGSingleband1) + val fullRaster = RasterRef(src, 0, None) val subExtent = sub(src.extent) - val subRaster = RasterRef(src, Option(subExtent)) + val subRaster = RasterRef(src, 0, Some(subExtent)) } import spark.implicits._ + implicit val enc = Encoders.tuple(Encoders.scalaInt, RasterRef.rrEncoder) describe("GetCRS Expression") { it("should read from RasterRef") { new Fixture { @@ -94,7 +96,6 @@ class RasterRefSpec extends TestEnvironment with TestData { new Fixture { val ds = Seq((1, RasterRefTile(fullRaster): Tile)).toDF("index", "ref") val dims = ds.select(GetDimensions($"ref")) - println(counter) assert(dims.count() === 1) assert(dims.first() !== null) } @@ -103,19 +104,18 @@ class RasterRefSpec extends TestEnvironment with TestData { new Fixture { val ds = Seq((1, RasterRefTile(subRaster): Tile)).toDF("index", "ref") val dims = ds.select(GetDimensions($"ref")) - println(counter) assert(dims.count() === 1) assert(dims.first() !== null) } } } - describe("GetExtent Expression") { + describe("GetExtent") { it("should read from RasterRef") { import spark.implicits._ new Fixture { val ds = Seq((1, fullRaster)).toDF("index", "ref") - val extent = ds.select(GetExtent($"ref")) + val extent = ds.select(rf_extent($"ref")) assert(extent.count() === 1) assert(extent.first() !== null) } @@ -124,7 +124,7 @@ class RasterRefSpec extends TestEnvironment with TestData { import spark.implicits._ new Fixture { val ds = Seq((1, subRaster)).toDF("index", "ref") - val extent = ds.select(GetExtent($"ref")) + val extent = ds.select(rf_extent($"ref")) assert(extent.count() === 1) assert(extent.first() !== null) } @@ -135,23 +135,18 @@ class RasterRefSpec extends TestEnvironment with TestData { it("should delay reading") { new Fixture { assert(subRaster.cellType === src.cellType) - assert(counter.reads === 0) } } it("should support subextents") { new Fixture { assert(subRaster.cols.toDouble === src.cols * 0.01 +- 2.0) assert(subRaster.rows.toDouble === src.rows * 0.01 +- 2.0) - assert(counter.reads === 0) //subRaster.tile.rescale(0, 255).renderPng().write("target/foo1.png") } } it("should be realizable") { new Fixture { - assert(counter.reads === 0) assert(subRaster.tile.statistics.map(_.dataCells) === Some(subRaster.cols * subRaster.rows)) - assert(counter.reads > 0) - println(counter) } } @@ -166,29 +161,52 @@ class RasterRefSpec extends TestEnvironment with TestData { val data = buf.toByteArray val in = new ObjectInputStream(new ByteArrayInputStream(data)) val recovered = in.readObject() - assert(subRaster === recovered) + subRaster should be (recovered) } } } - describe("CreateRasterRefs") { - it("should convert and expand RasterSource") { - new Fixture { - import spark.implicits._ - val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs($"src")) - assert(refs.count() > 1) + describe("RasterRef creation") { + it("should realize subiles of proper size") { + val src = RasterSource(remoteMODIS) + val dims = src + .layoutExtents(NOMINAL_TILE_DIMS) + .map(e => RasterRef(src, 0, Some(e))) + .map(_.dimensions) + .distinct + + forEvery(dims) { d => + d._1 should be <= NOMINAL_TILE_SIZE + d._2 should be <= NOMINAL_TILE_SIZE } } + } - it("should work with tile realization") { - new Fixture { - import spark.implicits._ - val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs(true, $"src")) - assert(refs.count() > 1) - } + describe("RasterSourceToRasterRefs") { + it("should convert and expand RasterSource") { + val src = RasterSource(remoteMODIS) + import spark.implicits._ + val df = Seq(src).toDF("src") + val refs = df.select(RasterSourceToRasterRefs(None, Seq(0), $"src")) + refs.count() should be (1) } + it("should properly realize subtiles") { + val src = RasterSource(remoteMODIS) + import spark.implicits._ + val df = Seq(src).toDF("src") + val refs = df.select(RasterSourceToRasterRefs(Some(NOMINAL_TILE_DIMS), Seq(0), $"src") as "proj_raster") + + refs.count() shouldBe > (1L) + + + val dims = refs.select(rf_dimensions($"proj_raster")).distinct().collect() + forEvery(dims) { r => + r.cols should be <=NOMINAL_TILE_SIZE + r.rows should be <=NOMINAL_TILE_SIZE + } + + dims.foreach(println) + } } } diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala new file mode 100644 index 000000000..1e62b95f5 --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala @@ -0,0 +1,182 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.ref + +import java.net.URI + +import org.locationtech.rasterframes._ +import geotrellis.vector.Extent +import org.apache.spark.sql.rf.RasterSourceUDT +import org.locationtech.rasterframes.model.TileDimensions + + +class RasterSourceSpec extends TestEnvironment with TestData { + def sub(e: Extent) = { + val c = e.center + val w = e.width + val h = e.height + Extent(c.x, c.y, c.x + w * 0.1, c.y + h * 0.1) + } + + describe("General RasterSource") { + it("should identify as UDT") { + assert(new RasterSourceUDT() === new RasterSourceUDT()) + } + val rs = RasterSource(getClass.getResource("/L8-B8-Robinson-IL.tiff").toURI) + it("should compute nominal tile layout bounds") { + val bounds = rs.layoutBounds(TileDimensions(65, 60)) + val agg = bounds.reduce(_ combine _) + agg should be (rs.gridBounds) + } + it("should compute nominal tile layout extents") { + val extents = rs.layoutExtents(TileDimensions(63, 63)) + val agg = extents.reduce(_ combine _) + agg should be (rs.extent) + } + it("should reassemble correct grid from extents") { + val dims = TileDimensions(63, 63) + val ext = rs.layoutExtents(dims).head + val bounds = rs.layoutBounds(dims).head + rs.rasterExtent.gridBoundsFor(ext) should be (bounds) + } + it("should compute layout extents from scene with fractional gsd") { + + val rs = RasterSource(remoteMODIS) + + val dims = rs.layoutExtents(NOMINAL_TILE_DIMS) + .map(e => rs.rasterExtent.gridBoundsFor(e, false)) + .map(b => (b.width, b.height)) + .distinct + forEvery(dims) { d => + d._1 should be <= NOMINAL_TILE_SIZE + d._2 should be <= NOMINAL_TILE_SIZE + } + } + } + + describe("HTTP RasterSource") { + it("should support metadata querying over HTTP") { + withClue("remoteCOGSingleband") { + val src = RasterSource(remoteCOGSingleband1) + assert(!src.extent.isEmpty) + } + withClue("remoteCOGMultiband") { + val src = RasterSource(remoteCOGMultiband) + assert(!src.extent.isEmpty) + } + } + it("should read sub-tile") { + withClue("remoteCOGSingleband") { + val src = RasterSource(remoteCOGSingleband1) + val raster = src.read(sub(src.extent)) + assert(raster.size > 0 && raster.size < src.size) + } + withClue("remoteCOGMultiband") { + val src = RasterSource(remoteCOGMultiband) + val raster = src.read(sub(src.extent)) + assert(raster.size > 0 && raster.size < src.size) + } + } + it("should Java serialize") { + import java.io._ + val src = RasterSource(remoteCOGSingleband1) + val buf = new java.io.ByteArrayOutputStream() + val out = new ObjectOutputStream(buf) + out.writeObject(src) + out.close() + + val data = buf.toByteArray + val in = new ObjectInputStream(new ByteArrayInputStream(data)) + val recovered = in.readObject().asInstanceOf[RasterSource] + assert(src.toString === recovered.toString) + } + } + describe("File RasterSource") { + it("should support metadata querying of file") { + val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toUri + val src = RasterSource(localSrc) + assert(!src.extent.isEmpty) + } + it("should interpret no scheme as file://"){ + val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toString + val schemelessUri = new URI(localSrc) + schemelessUri.getScheme should be (null) + val src = RasterSource(schemelessUri) + assert(!src.extent.isEmpty) + } + } + + if(GDALRasterSource.hasGDAL) { + describe("GDAL Rastersource") { + val gdal = GDALRasterSource(cogPath) + val jvm = JVMGeoTiffRasterSource(cogPath) + it("should compute the same metadata as JVM RasterSource") { + gdal.cellType should be(jvm.cellType) + } + it("should compute the same dimensions as JVM RasterSource") { + val dims = TileDimensions(128, 128) + gdal.extent should be(jvm.extent) + gdal.rasterExtent should be(jvm.rasterExtent) + gdal.cellSize should be(jvm.cellSize) + gdal.layoutBounds(dims) should contain allElementsOf jvm.layoutBounds(dims) + gdal.layoutExtents(dims) should contain allElementsOf jvm.layoutExtents(dims) + } + + + it("should support vsi file paths") { + val archivePath = geotiffDir.resolve("L8-archive.zip") + val archiveURI = URI.create("gdal://vsizip/" + archivePath.toString + "/L8-RGB-VA.tiff") + val gdal = GDALRasterSource(archiveURI) + + gdal.bandCount should be (3) + } + + it("should interpret no scheme as file://") { + val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toString + val schemelessUri = new URI(localSrc) + val gdal = GDALRasterSource(schemelessUri) + val jvm = JVMGeoTiffRasterSource(schemelessUri) + gdal.extent should be (jvm.extent) + gdal.cellSize should be(jvm.cellSize) + } + } + } + + describe("RasterSource tile construction") { + it("should read all tiles") { + val src = RasterSource(remoteMODIS) + + val subrasters = src.readAll() + + val collected = subrasters.map(_.extent).reduceLeft(_.combine(_)) + + assert(src.extent.xmin === collected.xmin +- 0.01) + assert(src.extent.ymin === collected.ymin +- 0.01) + assert(src.extent.xmax === collected.xmax +- 0.01) + assert(src.extent.ymax === collected.ymax +- 0.01) + + val totalCells = subrasters.map(_.size).sum + + assert(totalCells === src.size) + } + } +} diff --git a/datasource/build.sbt b/datasource/build.sbt deleted file mode 100644 index b42fe3d3f..000000000 --- a/datasource/build.sbt +++ /dev/null @@ -1,35 +0,0 @@ -moduleName := "rasterframes-datasource" - -libraryDependencies ++= Seq( - geotrellis("s3").value, - spark("core").value % Provided, - spark("mllib").value % Provided, - spark("sql").value % Provided -) - -initialCommands in console := """ - |import astraea.spark.rasterframes._ - |import geotrellis.raster._ - |import geotrellis.spark.io.kryo.KryoRegistrator - |import org.apache.spark.serializer.KryoSerializer - |import org.apache.spark.sql._ - |import org.apache.spark.sql.functions._ - |import astraea.spark.rasterframes.datasource.geotrellis._ - |import astraea.spark.rasterframes.datasource.geotiff._ - |implicit val spark = SparkSession.builder() - | .master("local[*]") - | .appName(getClass.getName) - | .config("spark.serializer", classOf[KryoSerializer].getName) - | .config("spark.kryoserializer.buffer.max", "500m") - | .config("spark.kryo.registrationRequired", "false") - | .config("spark.kryo.registrator", classOf[KryoRegistrator].getName) - | .getOrCreate() - | .withRasterFrames - |spark.sparkContext.setLogLevel("ERROR") - |import spark.implicits._ - | - |""".stripMargin - -cleanupCommands in console := """ - |spark.stop() - |""".stripMargin \ No newline at end of file diff --git a/datasource/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/datasource/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index 26a271f13..a44f6fccd 100644 --- a/datasource/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/datasource/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1,3 +1,5 @@ -astraea.spark.rasterframes.datasource.geotiff.DefaultSource -astraea.spark.rasterframes.datasource.geotrellis.DefaultSource -astraea.spark.rasterframes.datasource.geotrellis.GeoTrellisCatalog +org.locationtech.rasterframes.datasource.geotiff.GeoTiffDataSource +org.locationtech.rasterframes.datasource.geotrellis.GeoTrellisLayerDataSource +org.locationtech.rasterframes.datasource.geotrellis.GeoTrellisCatalog +org.locationtech.rasterframes.datasource.raster.RasterSourceDataSource +org.locationtech.rasterframes.datasource.geojson.GeoJsonDataSource diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala deleted file mode 100644 index 74acbbc98..000000000 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala +++ /dev/null @@ -1,132 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package astraea.spark.rasterframes.datasource.geotiff - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.datasource._ -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister, RelationProvider} -import org.apache.spark.sql.types.LongType -import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode, functions ⇒ F} -import _root_.geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tags, Tiled} -import _root_.geotrellis.raster.io.geotiff.compression._ -import _root_.geotrellis.raster.io.geotiff.tags.codes.ColorSpace - -/** - * Spark SQL data source over GeoTIFF files. - * @since 1/14/18 - */ -class DefaultSource extends DataSourceRegister - with RelationProvider with CreatableRelationProvider - with DataSourceOptions with LazyLogging { - def shortName() = DefaultSource.SHORT_NAME - - def path(parameters: Map[String, String]) = - uriParam(PATH_PARAM, parameters) - - def createRelation(sqlContext: SQLContext, parameters: Map[String, String]) = { - val pathO = path(parameters) - require(pathO.isDefined, "Valid URI 'path' parameter required.") - sqlContext.withRasterFrames - - val p = pathO.get - - if(p.getPath.contains("*")) { - val bandCount = parameters.get(DefaultSource.BAND_COUNT_PARAM).map(_.toInt).getOrElse(1) - GeoTiffCollectionRelation(sqlContext, p, bandCount) - } - else GeoTiffRelation(sqlContext, p) - } - - override def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], data: DataFrame): BaseRelation = { - val pathO = path(parameters) - require(pathO.isDefined, "Valid URI 'path' parameter required.") - require(pathO.get.getScheme == "file" || pathO.get.getScheme == null, "Currently only 'file://' destinations are supported") - sqlContext.withRasterFrames - - require(data.isRF, "GeoTIFF can only be constructed from a RasterFrame") - val rf = data.certify - - // If no desired image size is given, write at full size. - lazy val (fullResCols, fullResRows) = { - // get the layout size given that the tiles may be heterogenously sized - // first get any valid row and column in the spatial key structure - val sk = rf.select(SPATIAL_KEY_COLUMN).first() - - val tc = rf.tileColumns.head - - val c = rf - .where(SPATIAL_KEY_COLUMN("row") === sk.row) - .agg( - F.sum(tile_dimensions(tc)("cols") cast(LongType)) - ).first() - .getLong(0) - - val r = rf - .where(SPATIAL_KEY_COLUMN("col") === sk.col) - .agg( - F.sum(tile_dimensions(tc)("rows") cast(LongType)) - ).first() - .getLong(0) - - (c, r) - } - - val cols = numParam(DefaultSource.IMAGE_WIDTH_PARAM, parameters).getOrElse(fullResCols) - val rows = numParam(DefaultSource.IMAGE_HEIGHT_PARAM, parameters).getOrElse(fullResRows) - - require(cols <= Int.MaxValue && rows <= Int.MaxValue, s"Can't construct a GeoTIFF of size $cols x $rows. (Too big!)") - - // Should we really play traffic cop here? - if(cols.toDouble * rows * 64.0 > Runtime.getRuntime.totalMemory() * 0.5) - logger.warn(s"You've asked for the construction of a very large image ($cols x $rows), destined for ${pathO.get}. Out of memory error likely.") - - val tcols = rf.tileColumns - val raster = rf.toMultibandRaster(tcols, cols.toInt, rows.toInt) - - // We make some assumptions here.... eventually have column metadata encode this. - val colorSpace = tcols.size match { - case 3 | 4 ⇒ ColorSpace.RGB - case _ ⇒ ColorSpace.BlackIsZero - } - - val compress = parameters.get(DefaultSource.COMPRESS_PARAM).map(_.toBoolean).getOrElse(false) - val options = GeoTiffOptions(Tiled, if (compress) DeflateCompression else NoCompression, colorSpace) - val tags = Tags( - RFBuildInfo.toMap.filter(_._1.startsWith("rf")).mapValues(_.toString), - tcols.map(c ⇒ Map("RF_COL" -> c.columnName)).toList - ) - val geotiff = new MultibandGeoTiff(raster.tile, raster.extent, raster.crs, tags, options) - - logger.debug(s"Writing DataFrame to GeoTIFF ($cols x $rows) at ${pathO.get}") - geotiff.write(pathO.get.getPath) - GeoTiffRelation(sqlContext, pathO.get) - } -} - -object DefaultSource { - final val SHORT_NAME = "geotiff" - final val PATH_PARAM = "path" - final val IMAGE_WIDTH_PARAM = "imageWidth" - final val IMAGE_HEIGHT_PARAM = "imageWidth" - final val COMPRESS_PARAM = "compress" - final val BAND_COUNT_PARAM = "bandCount" -} diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala deleted file mode 100644 index 6e607c3b4..000000000 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package astraea.spark.rasterframes.datasource - -import java.net.URI - -import astraea.spark.rasterframes._ -import org.apache.spark.sql.{DataFrameReader, DataFrameWriter} -import shapeless.tag -import shapeless.tag.@@ - -/** - * Extension methods enabled by this module. - * - * @since 1/16/18 - */ -package object geotiff { - /** Tagged type construction for enabling type-safe extension methods for loading - * a RasterFrame in expected form. */ - type GeoTiffRasterFrameReader = DataFrameReader @@ GeoTiffRasterFrameReaderTag - trait GeoTiffRasterFrameReaderTag - - /** Adds `geotiff` format specifier to `DataFrameReader`. */ - implicit class DataFrameReaderHasGeoTiffFormat(val reader: DataFrameReader) { - def geotiff: GeoTiffRasterFrameReader = - tag[GeoTiffRasterFrameReaderTag][DataFrameReader](reader.format(DefaultSource.SHORT_NAME)) - } - - implicit class DataFrameWriterHasGeoTiffFormat[T](val writer: DataFrameWriter[T]) { - def geotiff: DataFrameWriter[T] = writer.format(DefaultSource.SHORT_NAME) - } - - /** Adds `loadRF` to appropriately tagged `DataFrameReader` */ - implicit class GeoTiffReaderWithRF(val reader: GeoTiffRasterFrameReader) { - def loadRF(path: URI): RasterFrame = reader.load(path.toASCIIString).asRF - } -} diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/DataSourceOptions.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/DataSourceOptions.scala similarity index 90% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/DataSourceOptions.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/DataSourceOptions.scala index 5baa8d67f..d620dd4fd 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/DataSourceOptions.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/DataSourceOptions.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.datasource +package org.locationtech.rasterframes.datasource /** * Key constants associated with DataFrameReader options for certain DataSource implementations. diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DOM.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/DOM.scala similarity index 92% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DOM.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/DOM.scala index 2dbcb7f0c..dfbbb92f3 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DOM.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/DOM.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2019 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,16 +15,17 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.geojson +package org.locationtech.rasterframes.datasource.geojson -import com.vividsolutions.jts.geom.{Envelope, Geometry} -import com.vividsolutions.jts.io.geojson.{GeoJsonReader, GeoJsonWriter} import geotrellis.vector.Extent -import spray.json._ +import org.locationtech.jts.geom.{Envelope, Geometry} +import org.locationtech.jts.io.geojson.{GeoJsonReader, GeoJsonWriter} import spray.json.DefaultJsonProtocol._ +import spray.json._ /** * Lightweight DOM for parsing GeoJSON feature sets. diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSource.scala similarity index 94% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSource.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSource.scala index f042fbd1c..1bda41cd7 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSource.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2019 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,13 +15,12 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.geojson +package org.locationtech.rasterframes.datasource.geojson -import astraea.spark.rasterframes.experimental.datasource.geojson.DOM._ -import com.vividsolutions.jts.geom.Geometry import org.apache.spark.annotation.Experimental import org.apache.spark.rdd.RDD import org.apache.spark.sql.jts.JTSTypes @@ -29,6 +28,8 @@ import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationP import org.apache.spark.sql.types.{DataTypes, StringType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row, SQLContext} import org.locationtech.geomesa.spark.jts._ +import org.locationtech.jts.geom.Geometry +import org.locationtech.rasterframes.datasource.geojson.DOM._ import spray.json.DefaultJsonProtocol._ import spray.json._ diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/package.scala similarity index 94% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/package.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/package.scala index 262c255d1..6c49d75bc 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geojson/package.scala @@ -19,7 +19,8 @@ * */ -package astraea.spark.rasterframes.experimental.datasource +package org.locationtech.rasterframes.datasource + import org.apache.spark.sql.DataFrameReader /** diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala similarity index 73% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala index 2f69d4425..3148a67d0 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionRelation.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,42 +15,35 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.datasource.geotiff +package org.locationtech.rasterframes.datasource.geotiff import java.net.URI -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.datasource.geotiff.GeoTiffCollectionRelation.Cols -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.util._ import geotrellis.proj4.CRS import geotrellis.spark.io.hadoop.HadoopGeoTiffRDD import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.hadoop.fs.Path import org.apache.spark.rdd.RDD -import org.apache.spark.sql.jts.JTSTypes import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.sources.{BaseRelation, PrunedScan} import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.sql.{Row, SQLContext} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.geotiff.GeoTiffCollectionRelation.Cols +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.util._ -/** - * - * - * @since 7/31/18 - */ +private[geotiff] case class GeoTiffCollectionRelation(sqlContext: SQLContext, uri: URI, bandCount: Int) extends BaseRelation with PrunedScan { override def schema: StructType = StructType(Seq( StructField(Cols.PATH, StringType, false), - StructField(EXTENT_COLUMN.columnName, CatalystSerializer[Extent].schema, nullable = true), - StructField(CRS_COLUMN.columnName, CatalystSerializer[CRS].schema, false) -// StructField(METADATA_COLUMN.columnName, -// DataTypes.createMapType(StringType, StringType, false) -// ) + StructField(EXTENT_COLUMN.columnName, schemaOf[Extent], nullable = true), + StructField(CRS_COLUMN.columnName, schemaOf[CRS], false) ) ++ ( if(bandCount == 1) Seq(StructField(Cols.TL, new TileUDT, false)) else for(b ← 1 to bandCount) yield StructField(Cols.TL + "_" + b, new TileUDT, nullable = true) @@ -63,14 +56,12 @@ case class GeoTiffCollectionRelation(sqlContext: SQLContext, uri: URI, bandCount val columnIndexes = requiredColumns.map(schema.fieldIndex) - - HadoopGeoTiffRDD.multiband(new Path(uri.toASCIIString), keyer, HadoopGeoTiffRDD.Options.DEFAULT) .map { case ((path, pe), mbt) ⇒ val entries = columnIndexes.map { case 0 ⇒ path - case 1 ⇒ CatalystSerializer[Extent].toRow(pe.extent) - case 2 ⇒ CatalystSerializer[CRS].toRow(pe.crs) + case 1 ⇒ pe.extent.toRow + case 2 ⇒ pe.crs.toRow case i if i > 2 ⇒ { if(bandCount == 1 && mbt.bandCount > 2) mbt.color() else mbt.band(i - 3) @@ -78,7 +69,6 @@ case class GeoTiffCollectionRelation(sqlContext: SQLContext, uri: URI, bandCount } Row(entries: _*) } - } } @@ -86,7 +76,7 @@ object GeoTiffCollectionRelation { object Cols { lazy val PATH = "path" lazy val CRS = "crs" - lazy val EX = BOUNDS_COLUMN.columnName + lazy val EX = GEOMETRY_COLUMN.columnName lazy val TL = TILE_COLUMN.columnName } } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala new file mode 100644 index 000000000..77781a781 --- /dev/null +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala @@ -0,0 +1,191 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource.geotiff + +import java.net.URI + +import _root_.geotrellis.proj4.CRS +import _root_.geotrellis.raster._ +import _root_.geotrellis.raster.io.geotiff.compression._ +import _root_.geotrellis.raster.io.geotiff.tags.codes.ColorSpace +import _root_.geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tags, Tiled} +import _root_.geotrellis.spark._ +import com.typesafe.scalalogging.LazyLogging +import org.apache.spark.sql._ +import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister, RelationProvider} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource._ +import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate.ProjectedRasterDefinition +import org.locationtech.rasterframes.expressions.aggregates.{ProjectedLayerMetadataAggregate, TileRasterizerAggregate} +import org.locationtech.rasterframes.model.{LazyCRS, TileDimensions} +import org.locationtech.rasterframes.util._ + +/** + * Spark SQL data source over GeoTIFF files. + */ +class GeoTiffDataSource + extends DataSourceRegister with RelationProvider with CreatableRelationProvider with DataSourceOptions with LazyLogging { + import GeoTiffDataSource._ + + def shortName() = GeoTiffDataSource.SHORT_NAME + + def createRelation(sqlContext: SQLContext, parameters: Map[String, String]) = { + require(parameters.path.isDefined, "Valid URI 'path' parameter required.") + sqlContext.withRasterFrames + + val p = parameters.path.get + + if (p.getPath.contains("*")) { + val bandCount = parameters.get(GeoTiffDataSource.BAND_COUNT_PARAM).map(_.toInt).getOrElse(1) + GeoTiffCollectionRelation(sqlContext, p, bandCount) + } else GeoTiffRelation(sqlContext, p) + } + + override def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], df: DataFrame): BaseRelation = { + require(parameters.path.isDefined, "Valid URI 'path' parameter required.") + val path = parameters.path.get + require(path.getScheme == "file" || path.getScheme == null, "Currently only 'file://' destinations are supported") + sqlContext.withRasterFrames + + val tileCols = df.tileColumns + + require(tileCols.nonEmpty, "Could not find any tile columns.") + + val raster = if (df.isAlreadyLayer) { + val layer = df.certify + val tlm = layer.tileLayerMetadata.merge + + // If no desired image size is given, write at full size. + val TileDimensions(cols, rows) = parameters.rasterDimensions + .getOrElse { + val actualSize = tlm.layout.toRasterExtent().gridBoundsFor(tlm.extent) + TileDimensions(actualSize.width, actualSize.height) + } + + // Should we really play traffic cop here? + if (cols.toDouble * rows * 64.0 > Runtime.getRuntime.totalMemory() * 0.5) + logger.warn( + s"You've asked for the construction of a very large image ($cols x $rows), destined for ${path}. Out of memory error likely.") + + layer.toMultibandRaster(tileCols, cols.toInt, rows.toInt) + } else { + require(parameters.crs.nonEmpty, "A destination CRS must be provided") + require(tileCols.nonEmpty, "need at least one tile column") + + // Grab CRS to project into + val destCRS = parameters.crs.get + + // Select the anchoring Tile, Extent and CRS columns + val (extCol, crsCol, tileCol) = { + // Favor "ProjectedRaster" columns + val prCols = df.projRasterColumns + if (prCols.nonEmpty) { + (rf_extent(prCols.head), rf_crs(prCols.head), rf_tile(prCols.head)) + } else { + // If no "ProjectedRaster" column, look for single Extent and CRS columns. + val crsCols = df.crsColumns + require(crsCols.size == 1, "Exactly one CRS column must be in DataFrame") + val extentCols = df.extentColumns + require(extentCols.size == 1, "Exactly one Extent column must be in DataFrame") + (extentCols.head, crsCols.head, tileCols.head) + } + } + + // Scan table and constuct what the TileLayerMetadata would be in the specified destination CRS. + val tlm: TileLayerMetadata[SpatialKey] = df + .select( + ProjectedLayerMetadataAggregate( + destCRS, + extCol, + crsCol, + rf_cell_type(tileCol), + rf_dimensions(tileCol) + )) + .first() + logger.debug(s"Contructed TileLayerMetadata: ${tlm.toString}") + + val c = ProjectedRasterDefinition(tlm) + + val config = parameters.rasterDimensions + .map { dims => + c.copy(totalCols = dims.cols, totalRows = dims.rows) + } + .getOrElse(c) + + val aggs = tileCols + .map(t => TileRasterizerAggregate(config, crsCol, extCol, rf_tile(t))("tile").as(t.columnName)) + + val agg = df.select(aggs: _*) + + val row = agg.first() + + val bands = for (i <- 0 until row.size) yield row.getAs[Tile](i) + + ProjectedRaster(MultibandTile(bands), tlm.extent, tlm.crs) + } + + val tags = Tags( + RFBuildInfo.toMap.filter(_._1.toLowerCase().contains("version")).mapValues(_.toString), + tileCols.map(c => Map("RF_COL" -> c.columnName)).toList + ) + + // We make some assumptions here.... eventually have column metadata encode this. + val colorSpace = tileCols.size match { + case 3 | 4 => ColorSpace.RGB + case _ => ColorSpace.BlackIsZero + } + + val tiffOptions = GeoTiffOptions(Tiled, if (parameters.compress) DeflateCompression else NoCompression, colorSpace) + + val geotiff = new MultibandGeoTiff(raster.tile, raster.extent, raster.crs, tags, tiffOptions) + + logger.debug(s"Writing DataFrame to GeoTIFF (${geotiff.cols} x ${geotiff.rows}) at ${path}") + geotiff.write(path.getPath) + GeoTiffRelation(sqlContext, path) + } +} + +object GeoTiffDataSource { + final val SHORT_NAME = "geotiff" + final val PATH_PARAM = "path" + final val IMAGE_WIDTH_PARAM = "imageWidth" + final val IMAGE_HEIGHT_PARAM = "imageHeight" + final val COMPRESS_PARAM = "compress" + final val CRS_PARAM = "crs" + final val BAND_COUNT_PARAM = "bandCount" + + private[geotiff] implicit class ParamsDictAccessors(val parameters: Map[String, String]) extends AnyVal { + def path: Option[URI] = uriParam(PATH_PARAM, parameters) + def compress: Boolean = parameters.get(COMPRESS_PARAM).exists(_.toBoolean) + def crs: Option[CRS] = parameters.get(CRS_PARAM).map(s => LazyCRS(s)) + def rasterDimensions: Option[TileDimensions] = { + numParam(IMAGE_WIDTH_PARAM, parameters) + .zip(numParam(IMAGE_HEIGHT_PARAM, parameters)) + .map { + case (cols, rows) => + require(cols <= Int.MaxValue && rows <= Int.MaxValue, s"Can't construct a GeoTIFF of size $cols x $rows. (Too big!)") + TileDimensions(cols.toInt, rows.toInt) + } + .headOption + } + } +} diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffRelation.scala similarity index 85% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffRelation.scala index 8503171c8..b08ebc830 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffRelation.scala @@ -15,15 +15,18 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotiff +package org.locationtech.rasterframes.datasource.geotiff import java.net.URI -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.util._ +import com.typesafe.scalalogging.LazyLogging import geotrellis.proj4.CRS import geotrellis.spark._ import geotrellis.spark.io._ @@ -66,8 +69,8 @@ case class GeoTiffRelation(sqlContext: SQLContext, uri: URI) extends BaseRelatio StructType(Seq( StructField(SPATIAL_KEY_COLUMN.columnName, skSchema, nullable = false, skMetadata), - StructField(EXTENT_COLUMN.columnName, CatalystSerializer[Extent].schema, nullable = true), - StructField(CRS_COLUMN.columnName, CatalystSerializer[CRS].schema, nullable = true), + StructField(EXTENT_COLUMN.columnName, schemaOf[Extent], nullable = true), + StructField(CRS_COLUMN.columnName, schemaOf[CRS], nullable = true), StructField(METADATA_COLUMN.columnName, DataTypes.createMapType(StringType, StringType, false) ) @@ -85,8 +88,7 @@ case class GeoTiffRelation(sqlContext: SQLContext, uri: URI) extends BaseRelatio val trans = tlm.mapTransform val metadata = info.tags.headTags - val extSer = CatalystSerializer[Extent] - val encodedCRS = CatalystSerializer[CRS].toRow(tlm.crs) + val encodedCRS = tlm.crs.toRow if(info.segmentLayout.isTiled) { // TODO: Figure out how to do tile filtering via the range reader. @@ -98,7 +100,7 @@ case class GeoTiffRelation(sqlContext: SQLContext, uri: URI) extends BaseRelatio val gb = trans.extentToBounds(pe.extent) val entries = columnIndexes.map { case 0 => SpatialKey(gb.colMin, gb.rowMin) - case 1 => extSer.toRow(pe.extent) + case 1 => pe.extent.toRow case 2 => encodedCRS case 3 => metadata case n => tiles.band(n - 4) @@ -107,16 +109,15 @@ case class GeoTiffRelation(sqlContext: SQLContext, uri: URI) extends BaseRelatio } } else { - logger.warn("GeoTIFF is not already tiled. In-memory read required: " + uri) + //logger.warn("GeoTIFF is not already tiled. In-memory read required: " + uri) val geotiff = HadoopGeoTiffReader.readMultiband(new Path(uri)) val rdd = sqlContext.sparkContext.makeRDD(Seq((geotiff.projectedExtent, Shims.toArrayTile(geotiff.tile)))) - rdd.tileToLayout(tlm) .map { case (sk, tiles) ⇒ val entries = columnIndexes.map { case 0 => sk - case 1 => extSer.toRow(trans.keyToExtent(sk)) + case 1 => trans.keyToExtent(sk).toRow case 2 => encodedCRS case 3 => metadata case n => tiles.band(n - 4) diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/package.scala new file mode 100644 index 000000000..75bdc7e76 --- /dev/null +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/package.scala @@ -0,0 +1,81 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource +import java.net.URI + +import org.apache.spark.sql.{DataFrameReader, DataFrameWriter} +import org.locationtech.rasterframes._ +import _root_.geotrellis.proj4.CRS +import shapeless.tag.@@ +import shapeless.tag + +package object geotiff { + /** Tagged type construction for enabling type-safe extension methods for loading + * a RasterFrameLayer from a single GeoTiff. */ + type GeoTiffRasterFrameReader = DataFrameReader @@ GeoTiffRasterFrameReaderTag + trait GeoTiffRasterFrameReaderTag + + /** Tagged type construction for enabling type-safe extension methods for writing + * a RasterFrame to a geotiff. */ + type GeoTiffRasterFrameWriter[T] = DataFrameWriter[T] @@ GeoTiffRasterFrameWriterTag + trait GeoTiffRasterFrameWriterTag + + /** Adds `geotiff` format specifier to `DataFrameReader`. */ + implicit class DataFrameReaderHasGeoTiffFormat(val reader: DataFrameReader) { + @deprecated("Use `raster` instead.", "7/1/2019") + def geotiff: GeoTiffRasterFrameReader = + tag[GeoTiffRasterFrameReaderTag][DataFrameReader]( + reader.format(GeoTiffDataSource.SHORT_NAME) + ) + } + + implicit class DataFrameWriterHasGeoTiffFormat[T](val writer: DataFrameWriter[T]) { + def geotiff: GeoTiffRasterFrameWriter[T] = + tag[GeoTiffRasterFrameWriterTag][DataFrameWriter[T]]( + writer.format(GeoTiffDataSource.SHORT_NAME) + ) + + def withDimensions(cols: Int, rows: Int): GeoTiffRasterFrameWriter[T] = + tag[GeoTiffRasterFrameWriterTag][DataFrameWriter[T]]( + writer + .option(GeoTiffDataSource.IMAGE_WIDTH_PARAM, cols) + .option(GeoTiffDataSource.IMAGE_HEIGHT_PARAM, rows) + ) + + def withCompression: GeoTiffRasterFrameWriter[T] = + tag[GeoTiffRasterFrameWriterTag][DataFrameWriter[T]]( + writer + .option(GeoTiffDataSource.COMPRESS_PARAM, true) + ) + def withCRS(crs: CRS): GeoTiffRasterFrameWriter[T] = + tag[GeoTiffRasterFrameWriterTag][DataFrameWriter[T]]( + writer + .option(GeoTiffDataSource.CRS_PARAM, crs.toProj4String) + ) + } + + /** Adds `loadLayer` to appropriately tagged `DataFrameReader` */ + implicit class GeoTiffReaderWithRF(val reader: GeoTiffRasterFrameReader) { + @deprecated("Use `raster` instead.", "7/1/2019") + def loadLayer(path: URI): RasterFrameLayer = reader.load(path.toASCIIString).asLayer + } +} diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala similarity index 93% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala index 5af7f1e3f..11edc1d5f 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalog.scala @@ -15,23 +15,23 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.net.URI -import astraea.spark.rasterframes -import astraea.spark.rasterframes.datasource.geotrellis.GeoTrellisCatalog.GeoTrellisCatalogRelation -import astraea.spark.rasterframes.util.time import geotrellis.spark.io.AttributeStore import org.apache.spark.annotation.Experimental import org.apache.spark.rdd.RDD import org.apache.spark.sql._ +import org.apache.spark.sql.functions._ import org.apache.spark.sql.rf.VersionShims import org.apache.spark.sql.sources._ -import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType +import org.locationtech.rasterframes.datasource.geotrellis.GeoTrellisCatalog.GeoTrellisCatalogRelation import spray.json.DefaultJsonProtocol._ import spray.json._ diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisLayerDataSource.scala similarity index 90% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisLayerDataSource.scala index 63e88e25a..d12ea1e17 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisLayerDataSource.scala @@ -15,14 +15,16 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.net.URI -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.datasource.DataSourceOptions +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.DataSourceOptions import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.spark.io.index.ZCurveKeyIndexMethod @@ -37,9 +39,9 @@ import scala.util.Try * DataSource over a GeoTrellis layer store. */ @Experimental -class DefaultSource extends DataSourceRegister +class GeoTrellisLayerDataSource extends DataSourceRegister with RelationProvider with CreatableRelationProvider with DataSourceOptions { - def shortName(): String = DefaultSource.SHORT_NAME + def shortName(): String = GeoTrellisLayerDataSource.SHORT_NAME /** * Create a GeoTrellis data source. @@ -79,8 +81,8 @@ class DefaultSource extends DataSourceRegister require(layerName.isDefined, s"'$LAYER_PARAM' parameter for raster layer name required.") require(zoom.isDefined, s"Integer '$ZOOM_PARAM' parameter for raster layer zoom level required.") - val rf = data.asRFSafely - .getOrElse(throw new IllegalArgumentException("Only a valid RasterFrame can be saved as a GeoTrellis layer")) + val rf = data.asLayerSafely + .getOrElse(throw new IllegalArgumentException("Only a valid RasterFrameLayer can be saved as a GeoTrellis layer")) val tileColumn = parameters.get(TILE_COLUMN_PARAM).map(c ⇒ rf(c)) @@ -110,6 +112,6 @@ class DefaultSource extends DataSourceRegister } } -object DefaultSource { +object GeoTrellisLayerDataSource { final val SHORT_NAME = "geotrellis" } diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala similarity index 93% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala index 918f43015..343f4683d 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2017-2018 Azavea & Astraea, Inc. + * Copyright 2017-2019 Astraea, Inc. & Azavea * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,31 +15,33 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.io.UnsupportedEncodingException import java.net.URI import java.sql.{Date, Timestamp} import java.time.{ZoneOffset, ZonedDateTime} -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.datasource.geotrellis.GeoTrellisRelation.{C, TileFeatureData} -import astraea.spark.rasterframes.datasource.geotrellis.TileFeatureSupport._ -import astraea.spark.rasterframes.rules.splitFilters -import astraea.spark.rasterframes.rules.SpatialFilters.{Contains ⇒ sfContains, Intersects ⇒ sfIntersects} -import astraea.spark.rasterframes.rules.SpatialRelationReceiver -import astraea.spark.rasterframes.rules.TemporalFilters.{BetweenDates, BetweenTimes} -import astraea.spark.rasterframes.util.SubdivideSupport._ -import astraea.spark.rasterframes.util._ -import com.vividsolutions.jts.geom +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.geotrellis.TileFeatureSupport._ +import org.locationtech.rasterframes.rules.splitFilters +import org.locationtech.rasterframes.rules.SpatialFilters.{Contains => sfContains, Intersects => sfIntersects} +import org.locationtech.rasterframes.rules.SpatialRelationReceiver +import org.locationtech.rasterframes.rules.TemporalFilters.{BetweenDates, BetweenTimes} +import org.locationtech.rasterframes.util.SubdivideSupport._ +import org.locationtech.rasterframes.util._ +import com.typesafe.scalalogging.LazyLogging +import org.locationtech.jts.geom import geotrellis.raster.{CellGrid, MultibandTile, Tile, TileFeature} import geotrellis.spark.io._ import geotrellis.spark.io.avro.AvroRecordCodec import geotrellis.spark.util.KryoWrapper import geotrellis.spark.{LayerId, Metadata, SpatialKey, TileLayerMetadata, _} -import geotrellis.util.{LazyLogging, _} +import geotrellis.util._ import geotrellis.vector._ import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord @@ -49,6 +51,7 @@ import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import org.apache.spark.sql.{Row, SQLContext, sources} +import org.locationtech.rasterframes.datasource.geotrellis.GeoTrellisRelation.{C, TileFeatureData} import scala.reflect.ClassTag import scala.reflect.runtime.universe._ @@ -348,6 +351,6 @@ object GeoTrellisRelation { lazy val TS = TIMESTAMP_COLUMN.columnName lazy val TL = TILE_COLUMN.columnName lazy val TF = TILE_FEATURE_DATA_COLUMN.columnName - lazy val EX = BOUNDS_COLUMN.columnName + lazy val EX = GEOMETRY_COLUMN.columnName } } diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/Layer.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/Layer.scala similarity index 85% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/Layer.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/Layer.scala index a659de2ff..9f90c96fd 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/Layer.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/Layer.scala @@ -15,16 +15,18 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.net.URI -import astraea.spark.rasterframes -import astraea.spark.rasterframes.encoders.DelegatingSubfieldEncoder +import org.locationtech.rasterframes.encoders.DelegatingSubfieldEncoder import geotrellis.spark.LayerId import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.locationtech.rasterframes /** * /** Connector between a GT `LayerId` and the path in which it lives. */ diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/MergeableData.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/MergeableData.scala similarity index 95% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/MergeableData.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/MergeableData.scala index 73d537866..34bd6536b 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/MergeableData.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/MergeableData.scala @@ -15,9 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis trait MergeableData[D] { def merge(l:D, r:D): D diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupport.scala similarity index 96% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupport.scala index 6691d6d51..67ea65510 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupport.scala @@ -15,11 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes.util._ import geotrellis.raster.crop.{Crop, TileCropMethods} import geotrellis.raster.mask.TileMaskMethods import geotrellis.raster.merge.TileMergeMethods diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/package.scala similarity index 79% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/package.scala index 545c1f236..c4a7dc425 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotrellis/package.scala @@ -15,24 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource +package org.locationtech.rasterframes.datasource import java.net.URI -import _root_.geotrellis.spark.LayerId -import astraea.spark.rasterframes.datasource.geotrellis.DefaultSource._ -import astraea.spark.rasterframes.{RasterFrame, _} import org.apache.spark.sql._ -import org.apache.spark.sql.functions.col -import shapeless.tag +import org.apache.spark.sql.functions._ +import _root_.geotrellis.spark.LayerId +import org.locationtech.rasterframes._ import shapeless.tag.@@ +import shapeless.tag -/** - * Extension methods for literate and type-safe loading of geotrellis layers. - * - * @since 1/12/18 - */ package object geotrellis extends DataSourceOptions { implicit val layerEncoder = Layer.layerEncoder @@ -40,11 +36,11 @@ package object geotrellis extends DataSourceOptions { def geotrellis_layer = col("layer").as[Layer] /** Tagged type construction for enabling type-safe extension methods for loading - * a RasterFrame from a GeoTrellis layer. */ + * a RasterFrameLayer from a GeoTrellis layer. */ type GeoTrellisRasterFrameReader = DataFrameReader @@ GeoTrellisRasterFrameReaderTag trait GeoTrellisRasterFrameReaderTag /** Tagged type construction for enabling type-safe extension methods for writing - * a RasterFrame to a GeoTrellis layer. */ + * a RasterFrameLayer to a GeoTrellis layer. */ type GeoTrellisRasterFrameWriter[T] = DataFrameWriter[T] @@ GeoTrellisRasterFrameWriterTag trait GeoTrellisRasterFrameWriterTag @@ -56,12 +52,12 @@ package object geotrellis extends DataSourceOptions { reader.format("geotrellis-catalog").load(base.toASCIIString) def geotrellis: GeoTrellisRasterFrameReader = - tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](reader.format(SHORT_NAME)) + tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](reader.format(GeoTrellisLayerDataSource.SHORT_NAME)) } implicit class DataFrameWriterHasGeotrellisFormat[T](val writer: DataFrameWriter[T]) { def geotrellis: GeoTrellisRasterFrameWriter[T] = - tag[GeoTrellisRasterFrameWriterTag][DataFrameWriter[T]](writer.format(SHORT_NAME)) + tag[GeoTrellisRasterFrameWriterTag][DataFrameWriter[T]](writer.format(GeoTrellisLayerDataSource.SHORT_NAME)) } implicit class GeoTrellisWriterAddLayer[T](val writer: GeoTrellisRasterFrameWriter[T]) { @@ -75,25 +71,25 @@ package object geotrellis extends DataSourceOptions { .option("path", layer.base.toASCIIString) } - /** Extension methods for loading a RasterFrame from a tagged `DataFrameReader`. */ + /** Extension methods for loading a RasterFrameLayer from a tagged `DataFrameReader`. */ implicit class GeoTrellisReaderWithRF(val reader: GeoTrellisRasterFrameReader) { def withTileSubdivisions(divs: Int): GeoTrellisRasterFrameReader = tag[GeoTrellisRasterFrameReaderTag][DataFrameReader]( - reader.option(TILE_SUBDIVISIONS_PARAM, divs) + reader.option(TILE_SUBDIVISIONS_PARAM, divs.toLong) ) def withNumPartitions(partitions: Int): GeoTrellisRasterFrameReader = tag[GeoTrellisRasterFrameReaderTag][DataFrameReader]( - reader.option(NUM_PARTITIONS_PARAM, partitions) + reader.option(NUM_PARTITIONS_PARAM, partitions.toLong) ) - def loadRF(uri: URI, id: LayerId): RasterFrame = + def loadLayer(uri: URI, id: LayerId): RasterFrameLayer = reader .option(LAYER_PARAM, id.name) .option(ZOOM_PARAM, id.zoom.toString) .load(uri.toASCIIString) - .asRF + .asLayer - def loadRF(layer: Layer): RasterFrame = loadRF(layer.base, layer.id) + def loadLayer(layer: Layer): RasterFrameLayer = loadLayer(layer.base, layer.id) } } diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala similarity index 92% rename from datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala rename to datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala index 6962e9a11..9a649bb94 100644 --- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala @@ -15,14 +15,14 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes import java.net.URI -import org.apache.spark.sql.sources.{And, Filter} - import scala.util.Try /** diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala new file mode 100644 index 000000000..6cea717ec --- /dev/null +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala @@ -0,0 +1,154 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource.raster + +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.util._ +import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationProvider} +import org.locationtech.rasterframes.model.TileDimensions + +class RasterSourceDataSource extends DataSourceRegister with RelationProvider { + import RasterSourceDataSource._ + override def shortName(): String = SHORT_NAME + override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { + val bands = parameters.bandIndexes + val tiling = parameters.tileDims + val lazyTiles = parameters.lazyTiles + val spec = parameters.pathSpec + val catRef = spec.fold(_.registerAsTable(sqlContext), identity) + RasterSourceRelation(sqlContext, catRef, bands, tiling, lazyTiles) + } +} + +object RasterSourceDataSource { + final val SHORT_NAME = "raster" + final val PATH_PARAM = "path" + final val PATHS_PARAM = "paths" + final val BAND_INDEXES_PARAM = "bandIndexes" + final val TILE_DIMS_PARAM = "tileDimensions" + final val CATALOG_TABLE_PARAM = "catalogTable" + final val CATALOG_TABLE_COLS_PARAM = "catalogColumns" + final val CATALOG_CSV_PARAM = "catalogCSV" + final val LAZY_TILES_PARAM = "lazyTiles" + + final val DEFAULT_COLUMN_NAME = PROJECTED_RASTER_COLUMN.columnName + + trait WithBandColumns { + def bandColumnNames: Seq[String] + } + /** Container for specifying raster paths. */ + case class RasterSourceCatalog(csv: String, bandColumnNames: String*) extends WithBandColumns { + def registerAsTable(sqlContext: SQLContext): RasterSourceCatalogRef = { + import sqlContext.implicits._ + val lines = csv + .split(Array('\n','\r')) + .map(_.trim) + .filter(_.nonEmpty) + + val dsLines = sqlContext.createDataset(lines) + val catalog = sqlContext.read + .option("header", "true") + .option("ignoreTrailingWhiteSpace", true) + .option("ignoreLeadingWhiteSpace", true) + .csv(dsLines) + + val tmpName = tmpTableName() + catalog.createOrReplaceTempView(tmpName) + + val cols = if (bandColumnNames.isEmpty) catalog.columns.toSeq + else bandColumnNames + + RasterSourceCatalogRef(tmpName, cols: _*) + } + } + + object RasterSourceCatalog { + def apply(singlebandPaths: Seq[String]): Option[RasterSourceCatalog] = + if (singlebandPaths.isEmpty) None + else { + val header = DEFAULT_COLUMN_NAME + val csv = header + "\n" + singlebandPaths.mkString("\n") + Some(new RasterSourceCatalog(csv, header)) + } + } + + /** Container for specifying where to select raster paths from. */ + case class RasterSourceCatalogRef(tableName: String, bandColumnNames: String*) extends WithBandColumns + + private[raster] + implicit class ParamsDictAccessors(val parameters: Map[String, String]) extends AnyVal { + def tokenize(csv: String): Seq[String] = csv.split(',').map(_.trim) + + def tileDims: Option[TileDimensions] = + parameters.get(TILE_DIMS_PARAM) + .map(tokenize(_).map(_.toInt)) + .map { case Seq(cols, rows) => TileDimensions(cols, rows)} + + def bandIndexes: Seq[Int] = parameters + .get(BAND_INDEXES_PARAM) + .map(tokenize(_).map(_.toInt)) + .getOrElse(Seq(0)) + + + def lazyTiles: Boolean = parameters + .get(LAZY_TILES_PARAM).forall(_.toBoolean) + + def catalog: Option[RasterSourceCatalog] = { + val paths = ( + parameters + .get(PATHS_PARAM) + .toSeq + .flatMap(_.split(Array('\n','\r'))) ++ + parameters + .get(RasterSourceDataSource.PATH_PARAM) + .toSeq + ).filter(_.nonEmpty) + + RasterSourceCatalog(paths) + .orElse(parameters + .get(CATALOG_CSV_PARAM) + .map(RasterSourceCatalog(_, catalogTableCols: _*)) + ) + } + + def catalogTableCols: Seq[String] = parameters + .get(CATALOG_TABLE_COLS_PARAM) + .map(tokenize(_).filter(_.nonEmpty).toSeq) + .getOrElse(Seq.empty) + + def catalogTable: Option[RasterSourceCatalogRef] = parameters + .get(CATALOG_TABLE_PARAM) + .map(p => RasterSourceCatalogRef(p, catalogTableCols: _*)) + + def pathSpec: Either[RasterSourceCatalog, RasterSourceCatalogRef] = { + (catalog, catalogTable) match { + case (Some(f), None) => Left(f) + case (None, Some(p)) => Right(p) + case (None, None) => throw new IllegalArgumentException( + s"Unable to interpret paths from: ${parameters.mkString("\n", "\n", "\n")}") + case _ => throw new IllegalArgumentException( + "Only one of a set of file paths OR a paths table column may be provided.") + } + } + } +} diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala new file mode 100644 index 000000000..6af519f56 --- /dev/null +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala @@ -0,0 +1,136 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource.raster + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.sources.{BaseRelation, TableScan} +import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.{DataFrame, Row, SQLContext} +import org.locationtech.rasterframes.datasource.raster.RasterSourceDataSource.RasterSourceCatalogRef +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.expressions.generators.{RasterSourceToRasterRefs, RasterSourceToTiles} +import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames +import org.locationtech.rasterframes.expressions.transformers.{RasterRefToTile, URIToRasterSource} +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +/** + * Constructs a Spark Relation over one or more RasterSource paths. + * @param sqlContext Query context + * @param catalogTable Specification of raster path sources + * @param bandIndexes band indexes to fetch + * @param subtileDims how big to tile/subdivide rasters info + */ +case class RasterSourceRelation( + sqlContext: SQLContext, + catalogTable: RasterSourceCatalogRef, + bandIndexes: Seq[Int], + subtileDims: Option[TileDimensions], + lazyTiles: Boolean +) extends BaseRelation with TableScan { + + lazy val inputColNames = catalogTable.bandColumnNames + + def pathColNames = inputColNames + .map(_ + "_path") + + def srcColNames = inputColNames + .map(_ + "_src") + + def refColNames = srcColNames + .flatMap(bandNames(_, bandIndexes)) + .map(_ + "_ref") + + def tileColNames = inputColNames + .flatMap(bandNames(_, bandIndexes)) + + lazy val extraCols: Seq[StructField] = { + val catalog = sqlContext.table(catalogTable.tableName) + catalog.schema.fields.filter(f => !catalogTable.bandColumnNames.contains(f.name)) + } + + override def schema: StructType = { + val tileSchema = schemaOf[ProjectedRasterTile] + val paths = for { + pathCol <- pathColNames + } yield StructField(pathCol, StringType, false) + val tiles = for { + tileColName <- tileColNames + } yield StructField(tileColName, tileSchema, true) + + StructType(paths ++ tiles ++ extraCols) + } + + override def buildScan(): RDD[Row] = { + import sqlContext.implicits._ + + // The general transformaion is: + // input -> path -> src -> ref -> tile + // Each step is broken down for readability + val inputs: DataFrame = sqlContext.table(catalogTable.tableName) + + // Basically renames the input columns to have the '_path' suffix + val pathsAliasing = for { + (input, path) <- inputColNames.zip(pathColNames) + } yield col(input).as(path) + + // Wraps paths in a RasterSource + val srcs = for { + (pathColName, srcColName) <- pathColNames.zip(srcColNames) + } yield URIToRasterSource(col(pathColName)) as srcColName + + // Add path columns + val withPaths = inputs + .select($"*" +: pathsAliasing: _*) + + // Path columns have to be manually pulled along through each step. Resolve columns once + // and reused with each select. + val paths = pathColNames.map(withPaths.apply) + + // Input columns along for the ride. + val extras = extraCols.map(f => inputs(f.name)) + + val df = if (lazyTiles) { + // Expand RasterSource into multiple columns per band, and multiple rows per tile + // There's some unintentional fragililty here in that the structure of the expression + // is expected to line up with our column structure here. + val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, srcs: _*) as refColNames + + // RasterSourceToRasterRef is a generator, which means you have to do the Tile conversion + // in a separate select statement (Query planner doesn't know how many columns ahead of time). + val refsToTiles = for { + (refColName, tileColName) <- refColNames.zip(tileColNames) + } yield RasterRefToTile(col(refColName)) as tileColName + + withPaths + .select(extras ++ paths :+ refs: _*) + .select(paths ++ refsToTiles ++ extras: _*) + } + else { + val tiles = RasterSourceToTiles(subtileDims, bandIndexes, srcs: _*) as tileColNames + withPaths + .select((paths :+ tiles) ++ extras: _*) + } + df.rdd + } +} diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/package.scala new file mode 100644 index 000000000..d85f435d2 --- /dev/null +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/package.scala @@ -0,0 +1,93 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource + +import java.net.URI +import java.util.UUID + +import org.apache.spark.sql.{DataFrame, DataFrameReader} +import shapeless.tag +import shapeless.tag.@@ +package object raster { + + private[raster] def tmpTableName() = UUID.randomUUID().toString.replace("-", "") + + trait RasterSourceDataFrameReaderTag + type RasterSourceDataFrameReader = DataFrameReader @@ RasterSourceDataFrameReaderTag + + /** Adds `raster` format specifier to `DataFrameReader`. */ + implicit class DataFrameReaderHasRasterSourceFormat(val reader: DataFrameReader) { + def raster: RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.format(RasterSourceDataSource.SHORT_NAME)) + } + + /** Adds option methods relevant to RasterSourceDataSource. */ + implicit class RasterSourceDataFrameReaderHasOptions(val reader: RasterSourceDataFrameReader) { + /** Set the zero-based band indexes to read. Defaults to Seq(0). */ + def withBandIndexes(bandIndexes: Int*): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.BAND_INDEXES_PARAM, bandIndexes.mkString(","))) + + def withTileDimensions(cols: Int, rows: Int): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.TILE_DIMS_PARAM, s"$cols,$rows") + ) + + /** Indicate if tile reading should be delayed until cells are fetched. Defaults to `true`. */ + def withLazyTiles(state: Boolean): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.LAZY_TILES_PARAM, state)) + + def fromCatalog(catalog: DataFrame, bandColumnNames: String*): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader] { + val tmpName = tmpTableName() + catalog.createOrReplaceTempView(tmpName) + reader + .option(RasterSourceDataSource.CATALOG_TABLE_PARAM, tmpName) + .option(RasterSourceDataSource.CATALOG_TABLE_COLS_PARAM, bandColumnNames.mkString(",")): DataFrameReader + } + + def fromCatalog(tableName: String, bandColumnNames: String*): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.CATALOG_TABLE_PARAM, tableName) + .option(RasterSourceDataSource.CATALOG_TABLE_COLS_PARAM, bandColumnNames.mkString(",")) + ) + + def fromCSV(catalogCSV: String, bandColumnNames: String*): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.CATALOG_CSV_PARAM, catalogCSV) + .option(RasterSourceDataSource.CATALOG_TABLE_COLS_PARAM, bandColumnNames.mkString(",")) + ) + + def from(newlineDelimPaths: String): RasterSourceDataFrameReader = + tag[RasterSourceDataFrameReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.PATHS_PARAM, newlineDelimPaths) + ) + + def from(paths: Seq[String]): RasterSourceDataFrameReader = + from(paths.mkString("\n")) + + def from(uris: Seq[URI])(implicit d: DummyImplicit): RasterSourceDataFrameReader = + from(uris.map(_.toASCIIString)) + } +} diff --git a/experimental/src/test/resources/buildings.geojson b/datasource/src/test/resources/buildings.geojson similarity index 100% rename from experimental/src/test/resources/buildings.geojson rename to datasource/src/test/resources/buildings.geojson diff --git a/experimental/src/test/resources/example.geojson b/datasource/src/test/resources/example.geojson similarity index 100% rename from experimental/src/test/resources/example.geojson rename to datasource/src/test/resources/example.geojson diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala deleted file mode 100644 index 3bdeecd81..000000000 --- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala +++ /dev/null @@ -1,117 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package astraea.spark.rasterframes.datasource.geotiff - -import java.nio.file.Paths - -import astraea.spark.rasterframes._ -import org.apache.spark.sql.functions._ - -/** - * @since 1/14/18 - */ -class GeoTiffDataSourceSpec - extends TestEnvironment with TestData { - - val cogPath = getClass.getResource("/LC08_RGB_Norfolk_COG.tiff").toURI - val nonCogPath = getClass.getResource("/L8-B8-Robinson-IL.tiff").toURI - val l8samplePath = getClass.getResource("/L8-B1-Elkton-VA.tiff").toURI - - describe("GeoTiff reading") { - - it("should read sample GeoTiff") { - val rf = spark.read - .geotiff - .loadRF(cogPath) - - assert(rf.count() > 10) - } - - it("should lay out tiles correctly"){ - - val rf = spark.read - .geotiff - .loadRF(cogPath) - - val tlm = rf.tileLayerMetadata.left.get - val gb = tlm.gridBounds - assert(gb.colMax > gb.colMin) - assert(gb.rowMax > gb.rowMin) - } - - it("should lay out tiles correctly for non-tiled tif") { - val rf = spark.read - .geotiff - .loadRF(nonCogPath) - - println(rf.count()) - rf.show(false) - - assert(rf.count() > 1) - - import org.apache.spark.sql.functions._ - logger.info( - rf.agg( - min(col("spatial_key.row")) as "rowmin", - max(col("spatial_key.row")) as "rowmax", - min(col("spatial_key.col")) as "colmin", - max(col("spatial_key.col")) as "colmax" - - ).first.toSeq.toString() - ) - val tlm = rf.tileLayerMetadata.left.get - val gb = tlm.gridBounds - assert(gb.rowMax > gb.rowMin) - assert(gb.colMax > gb.colMin) - - } - - it("should read in correctly check-summed contents") { - // c.f. TileStatsSpec -> computing statistics over tiles -> should compute tile statistics -> sum - val rf = spark.read.geotiff.loadRF(l8samplePath) - val expected = 309149454 // computed with rasterio - val result = rf.agg( - sum(tile_sum(rf("tile"))) - ).collect().head.getDouble(0) - - assert(result === expected) - } - - it("should write GeoTIFF RF to parquet") { - val rf = spark.read - .geotiff - .loadRF(cogPath) - assert(write(rf)) - } - - it("should write GeoTIFF") { - val rf = spark.read - .geotiff - .loadRF(cogPath) - - logger.info(s"Read extent: ${rf.tileLayerMetadata.merge.extent}") - - val out = Paths.get("target", "example-geotiff.tiff") - logger.info(s"Writing to $out") - noException shouldBe thrownBy { - rf.write.geotiff.save(out.toString) - } - } - } -} diff --git a/datasource/src/test/scala/examples/Creating.scala b/datasource/src/test/scala/examples/Creating.scala deleted file mode 100644 index 174bbaa3f..000000000 --- a/datasource/src/test/scala/examples/Creating.scala +++ /dev/null @@ -1,238 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import java.io.File -import java.nio.file.Files - -import geotrellis.raster.{Raster, Tile} - -/** - * Examples of creating RasterFrames - * - * @since 1/16/18 - */ -object Creating extends App { -/* -# Creating RasterFrames - -## Initialization - -There are a couple of setup steps necessary anytime you want to work with RasterFrames. the first is to import the API symbols into scope: - -*/ -// tut:silent -import astraea.spark.rasterframes._ -import org.apache.spark.sql._ - - -/* -Next, initialize the `SparkSession`, and call the `withRasterFrames` method on it: -*/ - -// tut:silent -implicit val spark = SparkSession.builder(). - master("local").appName("RasterFrames"). - getOrCreate(). - withRasterFrames - -/* -And, ss is standard Spark SQL practice, we import additional DataFrame support: -*/ - -// tut:silent -import spark.implicits._ -// tut:invisible -spark.sparkContext.setLogLevel("ERROR") - - -/* -Now we are ready to create a RasterFrame. - -## Reading a GeoTIFF - -The most straightforward way to create a `RasterFrame` is to read a [GeoTIFF](https://en.wikipedia.org/wiki/GeoTIFF) -file using a RasterFrame [`DataSource`](https://spark.apache.org/docs/latest/sql-programming-guide.html#data-sources) -designed for this purpose. - -First add the following import: -*/ - -import astraea.spark.rasterframes.datasource.geotiff._ -/* -(This is what adds the `.geotiff` method to `spark.read` below.) - -Then we use the `DataFrameReader` provided by `spark.read` to read the GeoTIFF: - */ - -// tut:book -val samplePath = new File("src/test/resources/LC08_RGB_Norfolk_COG.tiff") -val tiffRF = spark.read - .geotiff - .loadRF(samplePath.toURI) - -/* -Let's inspect the structure of what we get back: - */ - -// tut -tiffRF.printSchema() - -/* -As reported by Spark, RasterFrames extracts 6 columns from the GeoTIFF we selected. Some of these columns are dependent -on the contents of the source data, and some are are always available. Let's take a look at these in more detail. - -* `spatial_key`: GeoTrellis assigns a `SpatialKey` or a `SpaceTimeKey` to each tile, mapping it to the layer grid from - which it came. If it has a `SpaceTimeKey`, RasterFrames will split it into a `SpatialKey` and a `TemporalKey` (the - latter with column name `temporal_key`). -* `extent`: The bounding box of the tile in the tile's native CRS. -* `metadata`: The TIFF format header tags found in the file. -* `tile` or `tile_n` (where `n` is a band number): For singleband GeoTIFF files, the `tile` column contains the cell - data split into tiles. For multiband tiles, each column with `tile_` prefix contains each of the sources bands, - in the order they were stored. - -See the section [Inspecting a `RasterFrame`](#inspecting-a--code-rasterframe--code-) (below) for more details on accessing the RasterFrame's metadata. - */ - - -/* -## Reading a GeoTrellis Layer - -If your imagery is already ingested into a [GeoTrellis layer](https://docs.geotrellis.io/en/latest/guide/spark.html#writing-layers), -you can use the RasterFrames GeoTrellis DataSource. There are two parts to this GeoTrellis Layer support. The first -is the GeoTrellis Catalog DataSource, which lists the GeoTrellis layers available at a URI. The second part is the actual -RasterFrame reader for pulling a layer into a RasterFrame. - -Before we show how all of this works we need to have a GeoTrellis layer to work with. We can create one with the RasterFrame we -constructed above. - - */ -import astraea.spark.rasterframes.datasource.geotrellis._ - -val base = Files.createTempDirectory("rf-").toUri -val layer = Layer(base, "sample", 0) -tiffRF.write.geotrellis.asLayer(layer).save() - -/* -Now we can point our catalog reader at the base directory and see what was saved: -*/ - -val cat = spark.read.geotrellisCatalog(base) -cat.printSchema -cat.show() - -/* -As you can see, there's a lot of information stored in each row of the catalog. Most of this is associated with how the -layer is discretized. However, there may be other application specific metadata serialized with a layer that can be use -to filter the catalog entries or select a specific one. But for now, we're just going to load a RasterFrame in from the -catalog using a convenience function. - */ - -val firstLayer = cat.select(geotrellis_layer).first -val rfAgain = spark.read.geotrellis.loadRF(firstLayer) -rfAgain.show() - -/* -If you already know the `LayerId` of what you're wanting to read, you can bypass working with the catalog: - */ - -val anotherRF = spark.read.geotrellis.loadRF(layer) - -/* -## Using GeoTrellis APIs - -If you are used to working directly with the GeoTrellis APIs, there are a number of additional ways to create a `RasterFrame`, as enumerated in the sections below. - -First, some standard `import`s: -*/ - -// tut:silent -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import geotrellis.spark.io._ - -/* -### From `ProjectedExtent` - -The simplest mechanism for getting a RasterFrame is to use the `toRF(tileCols, tileRows)` extension method on `ProjectedRaster`. -*/ - -val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128) -rf.show(5, false) - -/* -### From `TileLayerRDD` - -Another option is to use a GeoTrellis [`LayerReader`](https://docs.geotrellis.io/en/latest/guide/tile-backends.html), -to get a `TileLayerRDD` for which there's also a `toRF` extension method. - -*/ - -/* -```scala -import geotrellis.spark._ -val tiledLayer: TileLayerRDD[SpatialKey] = ??? -val rf = tiledLayer.toRF -``` -*/ - -/* -## Inspecting a `RasterFrame` - -`RasterFrame` has a number of methods providing access to metadata about the contents of the RasterFrame. - -### Tile Column Names - -*/ - -//```tut:book -rf.tileColumns.map(_.toString) - -/* -### Spatial Key Column Name -*/ - -//```tut:book -rf.spatialKeyColumn.toString - -/* -### Temporal Key Column - -Returns an `Option[Column]` since not all RasterFrames have an explicit temporal dimension. -*/ - -//```tut:book -rf.temporalKeyColumn.map(_.toString) - -/* -### Tile Layer Metadata - -The Tile Layer Metadata defines how the spatial/spatiotemporal domain is discretized into tiles, and what the key bounds are. -*/ - -import spray.json._ -// NB: The `fold` is required because an `Either` is returned, depending on the key type. -rf.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint - - -//tut:invisible -spark.stop() - -} diff --git a/experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSourceTest.scala similarity index 89% rename from experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala rename to datasource/src/test/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSourceTest.scala index 425d91ab8..3d8ec9db3 100644 --- a/experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geojson/GeoJsonDataSourceTest.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2019 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,13 +15,13 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.geojson - -import astraea.spark.rasterframes.TestEnvironment +package org.locationtech.rasterframes.datasource.geojson import org.apache.spark.sql.types.{LongType, MapType} +import org.locationtech.rasterframes.TestEnvironment /** * Test rig for GeoJsonRelation. @@ -60,8 +60,7 @@ class GeoJsonDataSourceTest extends TestEnvironment { .option(GeoJsonDataSource.INFER_SCHEMA, true) .load(example2) - results.show() + results.count() should be (8) } } - } diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala similarity index 81% rename from datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala rename to datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala index 1d7237c5b..9b69fd89e 100644 --- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffCollectionDataSourceSpec.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,13 +15,15 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.datasource.geotiff +package org.locationtech.rasterframes.datasource.geotiff import java.io.{File, FilenameFilter} -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.TestEnvironment /** * @since 1/14/18 @@ -33,7 +35,7 @@ class GeoTiffCollectionDataSourceSpec it("shiould read a directory of files") { val df = spark.read - .geotiff + .format("geotiff") .load(geotiffDir.resolve("*.tiff").toString) val expected = geotiffDir.toFile.list(new FilenameFilter { override def accept(dir: File, name: String): Boolean = name.endsWith("tiff") @@ -41,7 +43,7 @@ class GeoTiffCollectionDataSourceSpec assert(df.select("path").distinct().count() === expected) - df.show(false) + // df.show(false) } } } diff --git a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala new file mode 100644 index 000000000..eb5e55b0c --- /dev/null +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala @@ -0,0 +1,244 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.locationtech.rasterframes.datasource.geotiff + +import java.nio.file.Paths + +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.{MultibandGeoTiff, SinglebandGeoTiff} +import geotrellis.vector.Extent +import org.locationtech.rasterframes._ +import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes.TestEnvironment + +/** + * @since 1/14/18 + */ +class GeoTiffDataSourceSpec + extends TestEnvironment with TestData { + + describe("GeoTiff reading") { + + it("should read sample GeoTiff") { + val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer + + assert(rf.count() > 10) + } + + it("should lay out tiles correctly") { + + val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer + + val tlm = rf.tileLayerMetadata.left.get + val gb = tlm.gridBounds + assert(gb.colMax > gb.colMin) + assert(gb.rowMax > gb.rowMin) + } + + it("should lay out tiles correctly for non-tiled tif") { + val rf = spark.read.format("geotiff").load(nonCogPath.toASCIIString).asLayer + + assert(rf.count() > 1) + + import org.apache.spark.sql.functions._ + logger.info( + rf.agg( + min(col("spatial_key.row")) as "rowmin", + max(col("spatial_key.row")) as "rowmax", + min(col("spatial_key.col")) as "colmin", + max(col("spatial_key.col")) as "colmax" + + ).first.toSeq.toString() + ) + val tlm = rf.tileLayerMetadata.left.get + val gb = tlm.gridBounds + assert(gb.rowMax > gb.rowMin) + assert(gb.colMax > gb.colMin) + + } + + it("should read in correctly check-summed contents") { + // c.f. TileStatsSpec -> computing statistics over tiles -> should compute tile statistics -> sum + val rf = spark.read.format("geotiff").load(l8B1SamplePath.toASCIIString).asLayer + val expected = 309149454 // computed with rasterio + val result = rf.agg( + sum(rf_tile_sum(rf("tile"))) + ).collect().head.getDouble(0) + + assert(result === expected) + } + } + + describe("GeoTiff writing") { + + it("should write GeoTIFF RF to parquet") { + val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer + assert(write(rf)) + } + + it("should write GeoTIFF from layer") { + val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer + + logger.info(s"Read extent: ${rf.tileLayerMetadata.merge.extent}") + + val out = Paths.get("target", "example-geotiff.tif") + logger.info(s"Writing to $out") + noException shouldBe thrownBy { + rf.write.format("geotiff").save(out.toString) + } + } + + it("should write unstructured raster") { + import spark.implicits._ + val df = spark.read.format("raster") + .option("tileDimensions", "32,32") // oddball + .load(nonCogPath.toASCIIString) // core L8-B8-Robinson-IL.tiff + + df.count() should be > 0L + + val crs = df.select(rf_crs($"proj_raster")).first() + + val out = Paths.get("target", "unstructured.tif").toString + + noException shouldBe thrownBy { + df.write.geotiff.withCRS(crs).save(out) + } + + val (inCols, inRows) = { + val id = sampleGeoTiff.imageData // inshallah same as nonCogPath + (id.cols, id.rows) + } + inCols should be (774) + inRows should be (500) //from gdalinfo + + val outputTif = SinglebandGeoTiff(out) + outputTif.imageData.cols should be (inCols) + outputTif.imageData.rows should be (inRows) + + // TODO check datatype, extent. + } + + it("should round trip unstructured raster from COG"){ + import spark.implicits._ + import org.locationtech.rasterframes.datasource.raster._ + + val df = spark.read.raster.withTileDimensions(64, 64).load(singlebandCogPath.toASCIIString) + + val resourceCols = 963 // from gdalinfo + val resourceRows = 754 + val resourceExtent = Extent(752325.0, 3872685.0, 781215.0, 3895305.0) + + df.count() should be > 0L + + val crs = df.select(rf_crs(col("proj_raster"))).first() + + val totalExtentRow = df.select(rf_extent($"proj_raster").alias("ext")) + .agg( + min($"ext.xmin").alias("xmin"), + min($"ext.ymin").alias("ymin"), + max($"ext.xmax").alias("xmax"), + max($"ext.ymax").alias("ymax") + ).first() + + val dfExtent = Extent(totalExtentRow.getDouble(0), totalExtentRow.getDouble(1), totalExtentRow.getDouble(2), totalExtentRow.getDouble(3)) + logger.info(s"Dataframe extent: ${dfExtent.toString()}") + + dfExtent shouldBe resourceExtent + + val out = Paths.get("target", "unstructured_cog.tif").toString + + noException shouldBe thrownBy { + df.write.geotiff.withCRS(crs).save(out) + } + + val (inCols, inRows, inExtent, inCellType) = { + val tif = readSingleband("LC08_B7_Memphis_COG.tiff") + val id = tif.imageData + (id.cols, id.rows, tif.extent, tif.cellType) + } + inCols should be (963) + inRows should be (754) //from gdalinfo + inExtent should be (resourceExtent) + + val outputTif = SinglebandGeoTiff(out) + outputTif.imageData.cols should be (inCols) + outputTif.imageData.rows should be (inRows) + outputTif.extent should be (resourceExtent) + outputTif.cellType should be (inCellType) + } + + it("should write GeoTIFF without layer") { + import org.locationtech.rasterframes.datasource.raster._ + val pr = col("proj_raster_b0") + val rf = spark.read.raster.withBandIndexes(0, 1, 2).load(rgbCogSamplePath.toASCIIString) + + val out = Paths.get("target", "example2-geotiff.tif") + logger.info(s"Writing to $out") + + withClue("explicit extent/crs") { + noException shouldBe thrownBy { + rf + .withColumn("extent", rf_extent(pr)) + .withColumn("crs", rf_crs(pr)) + .write.geotiff.withCRS(LatLng).save(out.toString) + } + } + + withClue("without explicit extent/crs") { + noException shouldBe thrownBy { + rf + .write.geotiff.withCRS(LatLng).save(out.toString) + } + } + withClue("with downsampling") { + noException shouldBe thrownBy { + rf + .write.geotiff + .withCRS(LatLng) + .withDimensions(128, 128) + .save(out.toString) + } + } + } + + def s(band: Int): String = + s"https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/" + + s"MCD43A4.A2019059.h11v08.006.2019072203257_B0${band}.TIF" + + it("shoud write multiband") { + import org.locationtech.rasterframes.datasource.raster._ + + val cat = s""" +red,green,blue +${s(1)},${s(4)},${s(3)} +""" + val scene = spark.read.raster.fromCSV(cat, "red", "green", "blue").load() + val out = Paths.get("target", "geotiff-overview.tif").toString + scene.write.geotiff + .withCRS(LatLng) + .withDimensions(256, 256) + .save(out) + + val outTif = MultibandGeoTiff(out) + outTif.bandCount should be (3) + } + } +} diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala similarity index 86% rename from datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala rename to datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala index e69a5414a..c409eb216 100644 --- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisCatalogSpec.scala @@ -15,17 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.io.File -import astraea.spark.rasterframes._ -import geotrellis.proj4.{CRS, LatLng, Sinusoidal} +import org.locationtech.rasterframes._ +import geotrellis.proj4.LatLng import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.spark.io.index.ZCurveKeyIndexMethod import org.apache.hadoop.fs.FileUtil +import org.locationtech.rasterframes.TestEnvironment import org.scalatest.BeforeAndAfter /** @@ -67,7 +70,7 @@ class GeoTrellisCatalogSpec .collect assert(layer.length === 2) - val lots = layer.map(sqlContext.read.geotrellis.loadRF).map(_.toDF).reduce(_ union _) + val lots = layer.map(sqlContext.read.geotrellis.loadLayer).map(_.toDF).reduce(_ union _) assert(lots.count === 60) } } diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala similarity index 79% rename from datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala rename to datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala index 009382639..ecd3351df 100644 --- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2017-2018 Azavea & Astraea, Inc. + * Copyright 2017-2019 Azavea & Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,17 +15,19 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis import java.io.File import java.sql.Timestamp import java.time.ZonedDateTime -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.datasource.DataSourceOptions -import astraea.spark.rasterframes.rules._ -import astraea.spark.rasterframes.util._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.DataSourceOptions +import org.locationtech.rasterframes.rules._ +import org.locationtech.rasterframes.util._ import geotrellis.proj4.LatLng import geotrellis.raster._ import geotrellis.raster.resample.NearestNeighbor @@ -40,9 +42,10 @@ import geotrellis.vector._ import org.apache.avro.generic._ import org.apache.avro.{Schema, SchemaBuilder} import org.apache.hadoop.fs.FileUtil -import org.apache.spark.sql.functions.{udf ⇒ sparkUdf} +import org.apache.spark.sql.functions.{udf => sparkUdf} import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.storage.StorageLevel +import org.locationtech.rasterframes.TestEnvironment import org.scalatest.{BeforeAndAfterAll, Inspectors} import scala.math.{max, min} @@ -88,7 +91,7 @@ class GeoTrellisDataSourceSpec outputDir.deleteOnExit() // Test layer writing via RF - testRdd.toRF.write.geotrellis.asLayer(layer).save() + testRdd.toLayer.write.geotrellis.asLayer(layer).save() val tfRdd = testRdd.map { case (k, tile) ⇒ val md = Map("col" -> k.col,"row" -> k.row) @@ -114,7 +117,7 @@ class GeoTrellisDataSourceSpec val tlfRdd = ContextRDD(tfRdd, testRdd.metadata) writer.write(tfLayer.id, tlfRdd, ZCurveKeyIndexMethod.byDay()) - //TestData.sampleTileLayerRDD.toRF.write.geotrellis.asLayer(sampleImageLayer).save() + //TestData.sampleTileLayerRDD.toLayer.write.geotrellis.asLayer(sampleImageLayer).save() val writer2 = LayerWriter(sampleImageLayer.base) val imgRDD = TestData.sampleTileLayerRDD writer2.write(sampleImageLayer.id, imgRDD, ZCurveKeyIndexMethod) @@ -123,13 +126,13 @@ class GeoTrellisDataSourceSpec describe("DataSource reading") { def layerReader = spark.read.geotrellis it("should read tiles") { - val df = layerReader.loadRF(layer) + val df = layerReader.loadLayer(layer) assert(df.count === tileCoordRange.length * tileCoordRange.length) } it("used produce tile UDT that we can manipulate") { - val df = layerReader.loadRF(layer) - .select(SPATIAL_KEY_COLUMN, tile_stats(TILE_COLUMN)) + val df = layerReader.loadLayer(layer) + .select(SPATIAL_KEY_COLUMN, rf_tile_stats(TILE_COLUMN)) assert(df.count() > 0) } @@ -138,7 +141,7 @@ class GeoTrellisDataSourceSpec val bbox = testRdd.metadata.layout .mapTransform(boundKeys.toGridBounds()) .jtsGeom - val wc = layerReader.loadRF(layer).withCenter() + val wc = layerReader.loadLayer(layer).withCenter() withClue("literate API") { val df = wc.where(CENTER_COLUMN intersects bbox) @@ -151,7 +154,7 @@ class GeoTrellisDataSourceSpec } it("should invoke Encoder[Extent]") { - val df = layerReader.loadRF(layer).withBounds() + val df = layerReader.loadLayer(layer).withGeometry() assert(df.count > 0) assert(df.first.length === 5) assert(df.first.getAs[Extent](2) !== null) @@ -159,7 +162,7 @@ class GeoTrellisDataSourceSpec it("should write to parquet") { //just should not throw - val df = layerReader.loadRF(layer) + val df = layerReader.loadLayer(layer) assert(write(df)) } } @@ -169,23 +172,23 @@ class GeoTrellisDataSourceSpec val expected = 2 val df = spark.read.geotrellis .withNumPartitions(expected) - .loadRF(layer) + .loadLayer(layer) assert(df.rdd.partitions.length === expected) } it("should respect partitions 20") { val expected = 20 val df = spark.read.geotrellis .withNumPartitions(expected) - .loadRF(layer) + .loadLayer(layer) assert(df.rdd.partitions.length === expected) } it("should respect subdivide 2") { val param = 2 - val df: RasterFrame = spark.read.geotrellis + val df: RasterFrameLayer = spark.read.geotrellis .withTileSubdivisions(param) - .loadRF(layer) + .loadLayer(layer) - val dims = df.select(tile_dimensions(df.tileColumns.head)("cols"), tile_dimensions(df.tileColumns.head)("rows")).first() + val dims = df.select(rf_dimensions(df.tileColumns.head)("cols"), rf_dimensions(df.tileColumns.head)("rows")).first() assert(dims.getAs[Int](0) === tileSize / param) assert(dims.getAs[Int](1) === tileSize / param) @@ -194,11 +197,11 @@ class GeoTrellisDataSourceSpec } it("should respect subdivide with TileFeature"){ val param = 2 - val rf: RasterFrame = spark.read.geotrellis + val rf: RasterFrameLayer = spark.read.geotrellis .withTileSubdivisions(param) - .loadRF(tfLayer) + .loadLayer(tfLayer) - val dims = rf.select(tile_dimensions(rf.tileColumns.head)("cols"), tile_dimensions(rf.tileColumns.head)("rows")) + val dims = rf.select(rf_dimensions(rf.tileColumns.head)("cols"), rf_dimensions(rf.tileColumns.head)("rows")) .first() assert(dims.getAs[Int](0) === tileSize / param) assert(dims.getAs[Int](1) === tileSize / param) @@ -213,14 +216,14 @@ class GeoTrellisDataSourceSpec .geotrellis .withNumPartitions(7) .withTileSubdivisions(subParam) - .loadRF(layer) + .loadLayer(layer) // is it partitioned correctly? assert(rf.rdd.partitions.length === 7) // is it subdivided? assert(rf.count === testRdd.count * subParam * subParam) - val dims = rf.select(tile_dimensions(rf.tileColumns.head)("cols"), tile_dimensions(rf.tileColumns.head)("rows")) + val dims = rf.select(rf_dimensions(rf.tileColumns.head)("cols"), rf_dimensions(rf.tileColumns.head)("rows")) .first() assert(dims.getAs[Int](0) === tileSize / subParam) assert(dims.getAs[Int](1) === tileSize / subParam) @@ -230,7 +233,7 @@ class GeoTrellisDataSourceSpec val subs = 4 val rf = spark.read.geotrellis .withTileSubdivisions(subs) - .loadRF(sampleImageLayer) + .loadLayer(sampleImageLayer) assert(rf.count === (TestData.sampleTileLayerRDD.count * subs * subs)) @@ -247,13 +250,13 @@ class GeoTrellisDataSourceSpec it("should throw on subdivide 5") { // only throws when an action is taken... - assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(5).loadRF(layer).cache) + assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(5).loadLayer(layer).cache) } it("should throw on subdivide 13") { - assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(13).loadRF(layer).cache) + assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(13).loadLayer(layer).cache) } it("should throw on subdivide -3") { - assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(-3).loadRF(layer).count) + assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(-3).loadLayer(layer).count) } } @@ -284,8 +287,8 @@ class GeoTrellisDataSourceSpec it("should support extent against a geometry literal") { val df: DataFrame = layerReader - .loadRF(layer) - .where(BOUNDS_COLUMN intersects pt1) + .loadLayer(layer) + .where(GEOMETRY_COLUMN intersects pt1) assert(numFilters(df) === 1) @@ -295,8 +298,8 @@ class GeoTrellisDataSourceSpec it("should support query with multiple geometry types") { // Mostly just testing that these evaluate without catalyst type errors. - forEvery(JTS.all) { g ⇒ - val query = layerReader.loadRF(layer).where(BOUNDS_COLUMN.intersects(g)) + forEvery(GeomData.all) { g ⇒ + val query = layerReader.loadLayer(layer).where(GEOMETRY_COLUMN.intersects(g)) .persist(StorageLevel.OFF_HEAP) assert(query.count() === 0) } @@ -308,8 +311,8 @@ class GeoTrellisDataSourceSpec val mkPtFcn = sparkUdf((_: Row) ⇒ { Point(-88, 60).jtsGeom }) val df = layerReader - .loadRF(layer) - .where(st_intersects(BOUNDS_COLUMN, mkPtFcn(SPATIAL_KEY_COLUMN))) + .loadLayer(layer) + .where(st_intersects(GEOMETRY_COLUMN, mkPtFcn(SPATIAL_KEY_COLUMN))) assert(numFilters(df) === 0) @@ -320,7 +323,7 @@ class GeoTrellisDataSourceSpec it("should support temporal predicates") { withClue("at now") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where(TIMESTAMP_COLUMN === Timestamp.valueOf(now.toLocalDateTime)) assert(numFilters(df) == 1) @@ -329,7 +332,7 @@ class GeoTrellisDataSourceSpec withClue("at earlier") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where(TIMESTAMP_COLUMN === Timestamp.valueOf(now.minusDays(1).toLocalDateTime)) assert(numFilters(df) === 1) @@ -338,7 +341,7 @@ class GeoTrellisDataSourceSpec withClue("between now") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where(TIMESTAMP_COLUMN betweenTimes (now.minusDays(1), now.plusDays(1))) assert(numFilters(df) === 1) @@ -347,7 +350,7 @@ class GeoTrellisDataSourceSpec withClue("between later") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where(TIMESTAMP_COLUMN betweenTimes (now.plusDays(1), now.plusDays(2))) assert(numFilters(df) === 1) @@ -358,10 +361,10 @@ class GeoTrellisDataSourceSpec it("should support nested predicates") { withClue("fully nested") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where( - ((BOUNDS_COLUMN intersects pt1) || - (BOUNDS_COLUMN intersects pt2)) && + ((GEOMETRY_COLUMN intersects pt1) || + (GEOMETRY_COLUMN intersects pt2)) && (TIMESTAMP_COLUMN === Timestamp.valueOf(now.toLocalDateTime)) ) @@ -373,8 +376,8 @@ class GeoTrellisDataSourceSpec withClue("partially nested") { val df = layerReader - .loadRF(layer) - .where((BOUNDS_COLUMN intersects pt1) || (BOUNDS_COLUMN intersects pt2)) + .loadLayer(layer) + .where((GEOMETRY_COLUMN intersects pt1) || (GEOMETRY_COLUMN intersects pt2)) .where(TIMESTAMP_COLUMN === Timestamp.valueOf(now.toLocalDateTime)) assert(numFilters(df) === 1) @@ -387,17 +390,17 @@ class GeoTrellisDataSourceSpec it("should support intersects with between times") { withClue("intersects first") { val df = layerReader - .loadRF(layer) - .where(BOUNDS_COLUMN intersects pt1) + .loadLayer(layer) + .where(GEOMETRY_COLUMN intersects pt1) .where(TIMESTAMP_COLUMN betweenTimes(now.minusDays(1), now.plusDays(1))) assert(numFilters(df) == 1) } withClue("intersects last") { val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where(TIMESTAMP_COLUMN betweenTimes(now.minusDays(1), now.plusDays(1))) - .where(BOUNDS_COLUMN intersects pt1) + .where(GEOMETRY_COLUMN intersects pt1) assert(numFilters(df) == 1) } @@ -405,10 +408,10 @@ class GeoTrellisDataSourceSpec withClue("untyped columns") { import spark.implicits._ val df = layerReader - .loadRF(layer) + .loadLayer(layer) .where($"timestamp" >= Timestamp.valueOf(now.minusDays(1).toLocalDateTime)) .where($"timestamp" <= Timestamp.valueOf(now.plusDays(1).toLocalDateTime)) - .where(st_intersects($"bounds", geomLit(pt1.jtsGeom))) + .where(st_intersects(GEOMETRY_COLUMN, geomLit(pt1.jtsGeom))) assert(numFilters(df) == 1) } @@ -417,19 +420,19 @@ class GeoTrellisDataSourceSpec it("should handle renamed spatial filter columns") { val df = layerReader - .loadRF(layer) - .where(BOUNDS_COLUMN intersects region.jtsGeom) - .withColumnRenamed(BOUNDS_COLUMN.columnName, "foobar") + .loadLayer(layer) + .where(GEOMETRY_COLUMN intersects region.jtsGeom) + .withColumnRenamed(GEOMETRY_COLUMN.columnName, "foobar") assert(numFilters(df) === 1) - assert(df.count > 0, df.printSchema) + assert(df.count > 0, df.schema.treeString) } it("should handle dropped spatial filter columns") { val df = layerReader - .loadRF(layer) - .where(BOUNDS_COLUMN intersects region.jtsGeom) - .drop(BOUNDS_COLUMN) + .loadLayer(layer) + .where(GEOMETRY_COLUMN intersects region.jtsGeom) + .drop(GEOMETRY_COLUMN) assert(numFilters(df) === 1) } @@ -437,18 +440,18 @@ class GeoTrellisDataSourceSpec describe("TileFeature support") { def layerReader = spark.read.geotrellis - it("should resolve TileFeature-based RasterFrame") { - val rf = layerReader.loadRF(tfLayer) + it("should resolve TileFeature-based RasterFrameLayer") { + val rf = layerReader.loadLayer(tfLayer) //rf.show(false) assert(rf.collect().length === testRdd.count()) } - it("should respect subdivideTile option on TileFeature RasterFrame") { + it("should respect subdivideTile option on TileFeature RasterFrameLayer") { val subParam = 4 - val rf = spark.read.option(TILE_SUBDIVISIONS_PARAM, subParam).geotrellis.loadRF(tfLayer) + val rf = spark.read.option(TILE_SUBDIVISIONS_PARAM, subParam).geotrellis.loadLayer(tfLayer) assert(rf.count === testRdd.count * subParam * subParam) - val dims = rf.select(tile_dimensions(rf.tileColumns.head)("cols"), tile_dimensions(rf.tileColumns.head)("rows")) + val dims = rf.select(rf_dimensions(rf.tileColumns.head)("cols"), rf_dimensions(rf.tileColumns.head)("rows")) .first() assert(dims.getAs[Int](0) === tileSize / subParam) assert(dims.getAs[Int](1) === tileSize / subParam) @@ -459,11 +462,11 @@ class GeoTrellisDataSourceSpec val rf = spark.read .option(TILE_SUBDIVISIONS_PARAM, subParam) .option(NUM_PARTITIONS_PARAM, 10) - .geotrellis.loadRF(tfLayer) + .geotrellis.loadLayer(tfLayer) // is it subdivided? assert(rf.count === testRdd.count * subParam * subParam) - val dims = rf.select(tile_dimensions(rf.tileColumns.head)("cols"), tile_dimensions(rf.tileColumns.head)("rows")) + val dims = rf.select(rf_dimensions(rf.tileColumns.head)("cols"), rf_dimensions(rf.tileColumns.head)("rows")) .first() assert(dims.getAs[Int](0) === tileSize / subParam) assert(dims.getAs[Int](1) === tileSize / subParam) diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala similarity index 94% rename from datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala rename to datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala index 5d475f263..0cf7e358c 100644 --- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala @@ -15,13 +15,15 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes.datasource.geotrellis +package org.locationtech.rasterframes.datasource.geotrellis -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.datasource.geotrellis.TileFeatureSupport._ -import astraea.spark.rasterframes.util.{WithCropMethods, WithMaskMethods, WithMergeMethods, WithPrototypeMethods} +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.geotrellis.TileFeatureSupport._ +import org.locationtech.rasterframes.util.{WithCropMethods, WithMaskMethods, WithMergeMethods, WithPrototypeMethods} import geotrellis.proj4.LatLng import geotrellis.raster.crop.Crop import geotrellis.raster.rasterize.Rasterizer @@ -32,6 +34,7 @@ import geotrellis.spark.tiling._ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD +import org.locationtech.rasterframes.TestEnvironment import org.scalatest.BeforeAndAfter import scala.reflect.ClassTag @@ -151,8 +154,6 @@ class TileFeatureSupportSpec extends TestEnvironment object TileFeatureSupportSpec { - import scala.language.implicitConversions - implicit class RichRandom(val rnd: scala.util.Random) extends AnyVal { def nextDouble(max: Double): Double = (rnd.nextInt * max) / Int.MaxValue.toDouble def nextOrderedPair(max:Double): (Double,Double) = (nextDouble(max),nextDouble(max)) match { diff --git a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSourceSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSourceSpec.scala new file mode 100644 index 000000000..a02c858e1 --- /dev/null +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSourceSpec.scala @@ -0,0 +1,314 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.datasource.raster +import geotrellis.raster.Tile +import org.apache.spark.sql.functions.{lit, udf, round} +import org.locationtech.rasterframes.{TestEnvironment, _} +import org.locationtech.rasterframes.datasource.raster.RasterSourceDataSource.{RasterSourceCatalog, _} +import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile +import org.locationtech.rasterframes.util._ + +class RasterSourceDataSourceSpec extends TestEnvironment with TestData { + import spark.implicits._ + + describe("DataSource parameter processing") { + def singleCol(paths: Iterable[String]) = { + val rows = paths.mkString(DEFAULT_COLUMN_NAME + "\n", "\n", "") + RasterSourceCatalog(rows, DEFAULT_COLUMN_NAME) + } + + it("should handle single `path`") { + val p = Map(PATH_PARAM -> "/usr/local/foo/bar.tif") + p.catalog should be (Some(singleCol(p.values))) + } + + it("should handle single `paths`") { + val p = Map(PATHS_PARAM -> "/usr/local/foo/bar.tif") + p.catalog should be (Some(singleCol(p.values))) + } + it("should handle multiple `paths`") { + val expected = Seq("/usr/local/foo/bar.tif", "/usr/local/bar/foo.tif") + val p = Map(PATHS_PARAM -> expected.mkString("\n\r", "\n\n", "\r")) + p.catalog should be (Some(singleCol(expected))) + } + it("should handle both `path` and `paths`") { + val expected1 = Seq("/usr/local/foo/bar.tif", "/usr/local/bar/foo.tif") + val expected2 = "/usr/local/barf/baz.tif" + val p = Map(PATHS_PARAM -> expected1.mkString("\n"), PATH_PARAM -> expected2) + p.catalog should be (Some(singleCol(expected1 :+ expected2))) + } + it("should parse tile dimensions") { + val p = Map(TILE_DIMS_PARAM -> "4, 5") + p.tileDims should be (Some(TileDimensions(4, 5))) + } + + it("should parse path table specification") { + val p = Map(CATALOG_TABLE_PARAM -> "catalogTable", CATALOG_TABLE_COLS_PARAM -> "path") + p.pathSpec should be (Right(RasterSourceCatalogRef("catalogTable", "path"))) + } + + it("should parse path table from CSV") { + val bands = Seq("B1", "B2", "B3") + val paths = Seq("/usr/local/foo/bar.tif", "/usr/local/bar/foo.tif", "/usr/local/barf/baz.tif") + val csv = + s""" + |${bands.mkString(",")} + |${paths.mkString(",")} + """.stripMargin.trim + val p = Map(CATALOG_CSV_PARAM -> csv) + p.pathSpec should be (Left(RasterSourceCatalog(csv))) + } + } + + describe("RasterSource as relation reading") { + val b = DEFAULT_COLUMN_NAME + + it("should default to a single band schema") { + val df = spark.read.raster.load(l8B1SamplePath.toASCIIString) + val tcols = df.tileColumns + tcols.length should be(1) + tcols.map(_.columnName) should contain(DEFAULT_COLUMN_NAME) + } + it("should support a multiband schema") { + val df = spark.read + .raster + .withBandIndexes(0, 1, 2) + .load(cogPath.toASCIIString) + val tcols = df.tileColumns + tcols.length should be(3) + tcols.map(_.columnName) should contain allElementsOf Seq("_b0", "_b1", "_b2").map(s => DEFAULT_COLUMN_NAME + s) + } + it("should read a multiband file") { + val df = spark.read + .raster + .withBandIndexes(0, 1, 2) + .load(cogPath.toASCIIString) + .cache() + df.schema.size should be (4) + // Test (roughly) we have three distinct but compabible bands + val stats = df.agg(rf_agg_stats($"${b}_b0") as "s0", rf_agg_stats($"${b}_b1") as "s1", rf_agg_stats($"${b}_b2") as "s2") + stats.select($"s0.data_cells" === $"s1.data_cells").as[Boolean].first() should be(true) + stats.select($"s0.data_cells" === $"s2.data_cells").as[Boolean].first() should be(true) + stats.select($"s0.mean" =!= $"s1.mean").as[Boolean].first() should be(true) + stats.select($"s0.mean" =!= $"s2.mean").as[Boolean].first() should be(true) + } + it("should read a single file") { + // Image is 1028 x 989 -> 9 x 8 tiles + val df = spark.read.raster + .withTileDimensions(128, 128) + .load(cogPath.toASCIIString) + + df.count() should be(math.ceil(1028.0 / 128).toInt * math.ceil(989.0 / 128).toInt) + + val dims = df.select(rf_dimensions($"$b").as[TileDimensions]).distinct().collect() + dims should contain allElementsOf + Seq(TileDimensions(4,128), TileDimensions(128,128), TileDimensions(128,93), TileDimensions(4,93)) + + df.select($"${b}_path").distinct().count() should be(1) + } + it("should read a multiple files with one band") { + val df = spark.read.raster + .from(Seq(cogPath, l8B1SamplePath, nonCogPath)) + .withTileDimensions(128, 128) + .load() + df.select($"${b}_path").distinct().count() should be(3) + df.schema.size should be(2) + } + it("should read a multiple files with heterogeneous bands") { + val df = spark.read.raster + .from(Seq(cogPath, l8B1SamplePath, nonCogPath)) + .withLazyTiles(false) + .withTileDimensions(128, 128) + .withBandIndexes(0, 1, 2, 3) + .load() + .cache() + df.select($"${b}_path").distinct().count() should be(3) + df.schema.size should be(5) + + df.select($"${b}_b0").count() should be (df.select($"${b}_b0").na.drop.count()) + df.select($"${b}_b1").na.drop.count() shouldBe <(df.count()) + df.select($"${b}_b1").na.drop.count() should be (df.select($"${b}_b2").na.drop.count()) + df.select($"${b}_b3").na.drop.count() should be (0) + } + + it("should read a set of coherent bands from multiple files from a CSV") { + val bands = Seq("B1", "B2", "B3") + val paths = Seq( + l8SamplePath(1).toASCIIString, + l8SamplePath(2).toASCIIString, + l8SamplePath(3).toASCIIString + ) + + val csv = + s""" + |${bands.mkString(",")} + |${paths.mkString(",")} + """.stripMargin.trim + + val df = spark.read.raster + .fromCSV(csv) + .withTileDimensions(128, 128) + .load() + + df.schema.size should be(6) + df.tileColumns.size should be (3) + df.select($"B1_path").distinct().count() should be (1) + } + + it("should read a set of coherent bands from multiple files in a dataframe") { + val bandPaths = Seq(( + l8SamplePath(1).toASCIIString, + l8SamplePath(2).toASCIIString, + l8SamplePath(3).toASCIIString)) + .toDF("B1", "B2", "B3") + .withColumn("foo", lit("something")) + + val df = spark.read.raster + .fromCatalog(bandPaths, "B1", "B2", "B3") + .withTileDimensions(128, 128) + .load() + + df.schema.size should be(7) + df.tileColumns.size should be (3) + df.select($"B1_path").distinct().count() should be (1) + + df.columns.contains("foo") should be (true) + df.select($"foo").distinct().count() should be (1) + df.select($"foo".as[String]).first() should be ("something") + + val diffStats = df.select(rf_tile_stats($"B1") =!= rf_tile_stats($"B2")).as[Boolean].collect() + diffStats.forall(identity) should be(true) + } + + it("should read a set of coherent bands from multiple files in a csv") { + def b(i: Int) = l8SamplePath(i).toASCIIString + + val csv = + s""" + |B1, B2, B3, foo + |${b(1)}, ${b(2)}, ${b(3)}, something + """.stripMargin + + val df = spark.read.raster + .fromCSV(csv, "B1", "B2", "B3") + .withTileDimensions(128, 128) + .load() + + df.schema.size should be(7) + df.tileColumns.size should be (3) + df.select($"B1_path").distinct().count() should be (1) + + df.columns.contains("foo") should be (true) + df.select($"foo").distinct().count() should be (1) + df.select($"foo".as[String]).first() should be ("something") + + val diffStats = df.select(rf_tile_stats($"B1") =!= rf_tile_stats($"B2")).as[Boolean].collect() + diffStats.forall(identity) should be(true) + } + + it("should support lazy and strict reading of tiles") { + val is_lazy = udf((t: Tile) => { + t.isInstanceOf[RasterRefTile] + }) + + val df1 = spark.read.raster + .withLazyTiles(true) + .load(l8SamplePath(1).toASCIIString) + + df1.select(is_lazy($"proj_raster.tile").as[Boolean]).first() should be (true) + + val df2 = spark.read.raster + .withLazyTiles(false) + .load(l8SamplePath(1).toASCIIString) + + df2.select(is_lazy($"proj_raster.tile").as[Boolean]).first() should be (false) + + } + } + + describe("RasterSource breaks up scenes into tiles") { + val modis_df = spark.read.raster + .withTileDimensions(128, 128) + .withLazyTiles(true) + .load(remoteMODIS.toASCIIString) + + val l8_df = spark.read.raster + .withTileDimensions(32, 33) + .withLazyTiles(true) + .load(remoteL8.toASCIIString) + + ignore("should have at most four tile dimensions reading MODIS; ignore until fix #242") { + val dims = modis_df.select(rf_dimensions($"proj_raster")).distinct().collect() + dims.length should be > (0) + dims.length should be <= (4) + } + + it("should have at most four tile dimensions reading landsat") { + val dims = l8_df.select(rf_dimensions($"proj_raster")).distinct().collect() + dims.length should be > (0) + dims.length should be <= (4) + } + + it("should provide MODIS tiles with requested size") { + val res = modis_df + .withColumn("dims", rf_dimensions($"proj_raster")) + .select($"dims".as[TileDimensions]).distinct().collect() + + forEvery(res) { r => + r.cols should be <=128 + r.rows should be <=128 + } + } + + it("should provide Landsat tiles with requested size") { + val dims = l8_df + .withColumn("dims", rf_dimensions($"proj_raster")) + .select($"dims".as[TileDimensions]).distinct().collect() + + forEvery(dims) { d => + d.cols should be <=32 + d.rows should be <=33 + } + } + + it("should have consistent tile resolution reading MODIS") { + val res = modis_df + .withColumn("ext", rf_extent($"proj_raster")) + .withColumn("dims", rf_dimensions($"proj_raster")) + .select(round(($"ext.xmax" - $"ext.xmin") / $"dims.cols", 5)) + .distinct().collect() + withClue(res.mkString("(", ", ", ")")) { + res.length should be(1) + } + } + + it("should have consistent tile resolution reading Landsat") { + val res = l8_df + .withColumn("ext", rf_extent($"proj_raster")) + .withColumn("dims", rf_dimensions($"proj_raster")) + .select(($"ext.xmax" - $"ext.xmin") / $"dims.cols") + .distinct().collect() + res.length should be (1) + } + } +} diff --git a/deployment/README.md b/deployment/README.md deleted file mode 100644 index 5e008b8a1..000000000 --- a/deployment/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# RasterFrames Jupyter Notebook Docker Container - -RasterFrames provides a Docker image with a Jupyter Notebook pre-configured with RasterFrames support for Python 3 and Scala Spylon kernels. - -## Quick start - -This will use the [latest image](https://hub.docker.com/r/s22s/rasterframes-notebooks/) published to Docker Hub. - -```bash -# Optionally pull the latest image. -$ docker pull s22s/rasterframes-notebooks - -# from root of the git repo -$ cd deployment/docker/jupyter -$ docker-compose up -``` - -## Custom run - -The `docker-compose` incantation automatically exposes port 8888 for the Jupyter Notebook and ports ports 4040-4044 for the Spark UI. - -The image can equivalently be run with: - - $ docker run -it --rm -p 8888:8888 -p 4040-4044:4040-4044 s22s/rasterframes-notebooks - -The `docker run` command can be changed to quickly customize the container. - -To mount a directory on the host machine (to load or save local files directly from Jupyter) add - - -v /some/host/folder/for/work:/home/jovyan/work - -to the command. - -Attach the notebook server to a different host port with - - -p 8630:8888 - -if you already have a notebook server running on port 8888. - -If you want to use a known password, use - -```bash -docker run -it --rm -p 8888:8888 -p 4040-4044:4040-4044 \ - s22s/rasterframes-notebooks \ - start-notebook.sh --NotebookApp.password='sha1:1c360e8dd3e1:946d17ef9e6b8cbb28c7bb0152329786918cc424' -``` - -Where the password sha is generated with [`notebook.auth.passwd`](https://jupyter-notebook.readthedocs.io/en/stable/public_server.html#preparing-a-hashed-password). - -Please see the `Dockerfile` and the `docker-compose.yml` file on GitHub ([here](https://github.com/locationtech/rasterframes/tree/develop/deployment/docker/jupyter)) as a starting point to customize your image and container. - - -## For Development - -To build the Docker image based on local development changes: - -```bash -# from the root of the repo -sbt deployment/rfDocker -``` - -## Base images - -This image is based on [jupyter/pyspark-notebook](https://hub.docker.com/r/jupyter/pyspark-notebook), with some -portions from [jupyter/all-spark-notebook](https://hub.docker.com/r/jupyter/all-spark-notebook). -Much more extensive instructions can be found at those locations. \ No newline at end of file diff --git a/deployment/build.sbt b/deployment/build.sbt deleted file mode 100644 index c76ef554b..000000000 --- a/deployment/build.sbt +++ /dev/null @@ -1,56 +0,0 @@ -import sbt.{IO, _} - -import scala.sys.process.Process - -moduleName := "rasterframes-deployment" - -val Docker = config("docker") -val Python = config("python") - - -lazy val rfDockerImageName = settingKey[String]("Name to tag Docker image with.") -rfDockerImageName := "s22s/rasterframes-notebooks" - -lazy val rfDocker = taskKey[Unit]("Build Jupyter Notebook Docker image with RasterFrames support.") -rfDocker := (Docker / packageBin).value - -lazy val runRFNotebook = taskKey[String]("Run RasterFrames Jupyter Notebook image") -runRFNotebook := { - val imageName = rfDockerImageName.value - val _ = rfDocker.value - Process(s"docker run -p 8888:8888 -p 4040:4040 $imageName").run() - imageName -} - -Docker / resourceDirectory := baseDirectory.value / "docker"/ "jupyter" - -Docker / target := target.value / "docker" - -Docker / mappings := { - val rezDir = (Docker / resourceDirectory).value - val files = (rezDir ** "*") pair Path.relativeTo(rezDir) - - val jar = (assembly in LocalProject("pyrasterframes")).value - val py = (packageBin in (LocalProject("pyrasterframes"), Python)).value - - files ++ Seq(jar -> jar.getName, py -> py.getName) -} - -def rfFiles = Def.task { - val destDir = (Docker / target).value - val filePairs = (Docker / mappings).value - IO.copy(filePairs.map { case (src, dst) ⇒ (src, destDir / dst) }) -} - -Docker / packageBin := { - val _ = rfFiles.value - val logger = streams.value.log - val staging = (Docker / target).value - val ver = (version in LocalRootProject).value - - logger.info(s"Running docker build in $staging") - val imageName = rfDockerImageName.value - Process("docker-compose build", staging).! - Process(s"docker tag $imageName:latest $imageName:$ver", staging).! - staging -} diff --git a/deployment/docker/jupyter/Dockerfile b/deployment/docker/jupyter/Dockerfile deleted file mode 100644 index ebf52fdac..000000000 --- a/deployment/docker/jupyter/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -FROM jupyter/pyspark-notebook:92fe05d1e7e5 - -MAINTAINER Astraea, Inc. - -ENV RF_LIB_LOC /usr/lib -ENV RF_JAR $RF_LIB_LOC/rasterframes.jar -ENV PY_RF_ZIP $RF_LIB_LOC/pyrasterframes.zip - -USER root - -RUN echo "spark.driver.extraClassPath $RF_JAR" >> /usr/local/spark/conf/spark-defaults.conf && \ - echo "spark.executor.extraClassPath $RF_JAR" >> /usr/local/spark/conf/spark-defaults.conf - -EXPOSE 4040 4041 4042 4043 4044 - -ENV SPARK_OPTS $SPARK_OPTS \ - --py-files $PY_RF_ZIP \ - --jars $RF_JAR \ - --driver-class-path $RF_JAR \ - --conf spark.executor.extraClassPath=$RF_JAR - -ENV PYTHONPATH $PYTHONPATH:$PY_RF_ZIP - - -#================================ -# Copied from all-spark-notebook -#================================ - -# TODO: resolve the issue that toree has with --py-files, above (it does not like .zips and -# TODO: the kernel will not start) -# Apache Toree kernel -#RUN pip install --no-cache-dir \ -# https://dist.apache.org/repos/dist/dev/incubator/toree/0.2.0-incubating-rc5/toree-pip/toree-0.2.0.tar.gz \ -# && \ -# jupyter toree install --sys-prefix && \ -# rm -rf /home/$NB_USER/.local && \ -# fix-permissions $CONDA_DIR && \ -# fix-permissions /home/$NB_USER - -# Spylon-kernel -RUN conda install --quiet --yes 'spylon-kernel=0.4*' && \ - conda clean -tipsy && \ - python -m spylon_kernel install --sys-prefix - -# Sphinx (for Notebook->html) -RUN conda install --quiet --yes \ - sphinx nbsphinx - -# Cleanup pip residuals -RUN rm -rf /home/$NB_USER/.local && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -# Do these after the standard environment setup -# since these change more regularly. -COPY *.zip $PY_RF_ZIP -COPY *.jar $RF_JAR - -RUN chown -R $NB_UID:$NB_GID $HOME - -USER $NB_UID - -# RUN pip install guzzle_sphinx_theme - -# TODO: This repo can change regularly without docker knowing that the -# TODO: Layer this command is written in has become stale. Need to either -# TODO: clone a specific revision that we manually update, or keep this -# TODO: last, assuming the prior commends will be detected as stale. -RUN git clone http://github.com/s22s/rasterframes-book && ln -s rasterframes-book/Python/samples diff --git a/deployment/docker/jupyter/README.md b/deployment/docker/jupyter/README.md deleted file mode 100644 index 815b78d8e..000000000 --- a/deployment/docker/jupyter/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# RasterFrames Jupyter Notebook - -Please visit `rasterframe-book` directory for example notebooks. \ No newline at end of file diff --git a/deployment/docker/jupyter/docker-compose.yml b/deployment/docker/jupyter/docker-compose.yml deleted file mode 100644 index 29f311c0f..000000000 --- a/deployment/docker/jupyter/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3' - -services: - rasterframes-notebooks: - build: . - image: s22s/rasterframes-notebooks - ports: - # jupyter notebook port - - "8888:8888" - # spark UI ports - - "4040:4040" - - "4041:4041" - - "4042:4042" - - "4043:4043" - - "4044:4044" -# To save locally at './work' from the container: -# volumes: -# - ./work:/home/jovyan/work \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..0c4925ba1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,128 @@ +# RasterFrames Documentation + +The conceptual principles to consider when writing RasterFrames users' documentation are covered in [Documentation Principles](documentation-principles.md). This document covers the mechanics of writing, evaluating, and building the documentation during the writing process. + +## Organization + +The documentation build is a two step process, whereby two sources (three if API docs are included) are merged together and converted into a static HTML website. They are: + +* Technical content and Python examples: `/pyrasterframes/src/main/python/docs` +* Global documentation assets and Scala specific content: `/docs/src/main/paradox` + +The build constructs in `/docs` are (due to legacy reasons) the top-level mechanisms of bringing it all together, but the meat of the content is in `/pyrasterframes/...`, and will be the focus of most of this document. + +## Prerequisites + +* [`sbt`](https://www.scala-sbt.org/) +* Python 3 +* Markdown editor. [Visual Studio Code](https://code.visualstudio.com/) with [`language-weave` extension](https://marketplace.visualstudio.com/items?itemName=jameselderfield.language-weave) is one option. [Atom](https://atom.io/) is another which might actually have better support for evaluating code in Markdown, but I've not tried it. + +> Note: If you're using Visual Studio Code, you can associate the `.pymd` with the `language-weave` plugin by adding this to your `settings.json` file. + +```json +"files.associations": { + "*.pymd": "pweave_md" +} +``` + +## Building the docs + +To build the static site locally: + + sbt makeSite + +The site will be at `/docs/target/site/index.html`. + + +To start an interactive server running the docs: + + sbt previewSite + +The sbt server log a message with an address to view the site. + +## Content Development Process + +Start with one of the existing files in `/pyrasterframes/src/main/python/docs` as a template. [`local-algebra.pymd`](../pyrasterframes/src/main/python/docs/local-algebra.pymd) is a good example. If the content will have code blocks you want evaluated an results injected into the output, use the file extension `.pymd`. If the content doesn't use evaluatable code blocks, use `.md`. + +All `.pymd` files are processed with a tool called [Pweave](http://mpastell.com/pweave), which produces a regular Markdown file where identified code blocks are evaluated and their results (optionally) included in the text. Matplot lib is supported! It is much like `knitr` in the R community. If we run into issues with Pweave, we can also consider [`knitpy`](https://github.com/jankatins/knitpy) or [`codebraid`](https://github.com/gpoore/codebraid). Codebraid looks particularly powerful, so we may think to transition to it. + +Pweave has a number of [code chunk options](http://mpastell.com/pweave/chunks.html) for controlling the output. Refer to those documents on details, and experiment a little to see what conveys your intent best. + +To set up an environment whereby you can easily test/evaluate your code blocks during writing: + +1. Run `sbt` from the `` directory. You should get output that looks something like: + ``` + $ sbt + ... + [info] Loading settings for project pyrasterframes from build.sbt ... + [info] Loading settings for project rf-notebook from build.sbt ... + [info] Set current project to RasterFrames (in build file:/) + sbt:RasterFrames> + ``` +2. The first time you check out the code, or whenever RasterFrames code is updated, you need to build the project artifacts so they are available for Pweave. Some docs also refer to test resources, so the easiest way to do it is to run the unit tests. + ``` + sbt:RasterFrames> pyrasterframes/test + [info] Compiling 4 Scala sources to /core/target/scala-2.11/classes ... + ... lots of noise ... + [info] PyRasterFrames assembly written to '/pyrasterframes/target/python/deps/jars/pyrasterframes-assembly-0.8.0-SNAPSHOT.jar' + [info] Synchronizing 44 files to '/pyrasterframes/target/python' + [info] Running 'python setup.py build bdist_wheel' in '/pyrasterframes/target/python' + ... more noise ... + [info] Python .whl file written to '/pyrasterframes/target/python/dist/pyrasterframes-0.8.0.dev0-py2.py3-none-any.whl' + [success] Total time: 83 s, completed Jul 5, 2019 12:25:48 PM + sbt:RasterFrames> + ``` +3. To evaluate all the `.pymd` files, run: + ``` + sbt:RasterFrames> pyrasterframes/pySetup pweave + ``` + To build the artifact (step 1) and evaluate all the `.pymd` files, you can run: + ``` + sbt:RasterFrames> pyrasterframes/doc + ``` + There's a command alias for this last step: `pyDocs`. +4. To evaluate a single `.pymd` file, you pass the `-s` option and the filename relative to the `pyraterframes/src/main/python` directory. You can also specify the output [format](http://mpastell.com/pweave/formats.html) with the `-f` argument. + ``` + sbt:RasterFrames> pyrasterframes/pySetup pweave -s docs/getting-started.pymd + [info] Synchronizing 44 files to '/pyrasterframes/target/python' + [info] Running 'python setup.py pweave -s docs/getting-started.pymd' in '/pyrasterframes/target/python' + running pweave + -------------------------------------------------- + Running getting-started + -------------------------------------------------- + status + status + Processing chunk 1 named None from line 14 + ... + Weaved docs/getting-started.pymd to docs/getting-started.md + ``` +5. The _output_ Markdown files are written to `/pyrasterframes/target/python/docs`. _Note_: don't edit any files in the `pyrasterframes/target` directory... they will get overwritten each time `sbt` runs a command. +6. During content development it's sometimes helpful to see the output rendered as basic HTML. To do this, add the `-f html` option to the pweave command: + ``` + sbt:RasterFrames> pyrasterframes/pySetup pweave -f html -s docs/getting-started.pymd + [info] Synchronizing 54 files to '/pyrasterframes/target/python' + [info] Running 'python setup.py pweave -f html -s docs/getting-started.pymd' in '/pyrasterframes/target/python' + running pweave + -------------------------------------------------- + Running getting-started + -------------------------------------------------- + ... + Weaved docs/getting-started.pymd to docs/getting-started.html + ``` + Note: This feature requires `pandoc` to be installed. +7. To build all the documentation and convert to a static html site, run: + ```bash + sbt makeSite + ``` + Results will be found in `/docs/target/site`. + +## Notebooks + +The `rf-notebooks` sub-project creates a Docker image with Jupyter Notebooks pre-configured with RasterFrames. Any `.pymd` file under `.../python/docs/` is converted to an evaluated Jupyter Notebook and included as a part of the build. + +## Submission Process + +Submit new and updated documentation as a PR against locationtech/rasterframes. Make sure you've signed the Eclipse Foundation ECA and you ["Signed-off-by:"](https://stackoverflow.com/questions/1962094/what-is-the-sign-off-feature-in-git-for) each commit in the PR. The "Signed-off-by" email address needs to be the exact same one as registered with the [Eclipse Foundation](https://wiki.eclipse.org/Development_Resources/Contributing_via_Git). + +If you are using circle CI, the circle configuration is set up to build the docs with `sbt makeSite` for branch names matching `feature/.*docs.*` or `docs/.*` + diff --git a/docs/build.sbt b/docs/build.sbt index 3025e7c75..59f734a48 100644 --- a/docs/build.sbt +++ b/docs/build.sbt @@ -1,41 +1,79 @@ -import com.typesafe.sbt.SbtGit.git +// task to create documentation PDF +lazy val makePDF = taskKey[File]("Build PDF version of documentation") +lazy val pdfFileName = settingKey[String]("Name of the PDF file generated") +pdfFileName := s"RasterFrames-Users-Manual-${version.value}.pdf" -enablePlugins(SiteScaladocPlugin, ParadoxPlugin, TutPlugin, GhpagesPlugin, ScalaUnidocPlugin) +makePDF := { + import scala.sys.process._ -name := "rasterframes-docs" + // Get the python source directory configured in the root project. + val base = (Compile / paradox / sourceDirectories).value.find(_.toString.contains("python")).head -libraryDependencies ++= Seq( - spark("mllib").value % Tut, - spark("sql").value % Tut -) + // Hard coded lacking any simple way of determining order. + val files = Seq( + "index.md", + "description.md", + "concepts.md", + "getting-started.md", + "raster-io.md", + "raster-catalogs.md", + "raster-read.md", + "raster-write.md", + "vector-data.md", + "raster-processing.md", + "local-algebra.md", + "nodata-handling.md", + "aggregation.md", + "time-series.md", + "machine-learning.md", + "unsupervised-learning.md", + "supervised-learning.md", + "numpy-pandas.md", + "languages.md", + "reference.md" + ).map(base ** _).flatMap(_.get) -git.remoteRepo := "git@github.com:locationtech/rasterframes.git" -apiURL := Some(url("http://rasterframes.io/latest/api")) -autoAPIMappings := true -ghpagesNoJekyll := true + val log = streams.value.log + log.info("Section ordering:") + files.foreach(f => log.info(" - " + f.getName)) -ScalaUnidoc / siteSubdirName := "latest/api" -paradox / siteSubdirName := "." + val work = target.value / "makePDF" + work.mkdirs() -addMappingsToSiteDir(ScalaUnidoc / packageDoc / mappings, ScalaUnidoc / siteSubdirName) -addMappingsToSiteDir(Compile / paradox / mappings, paradox / siteSubdirName) + val prepro = files.zipWithIndex.map { case (f, i) ⇒ + val dest = work / f"$i%02d-${f.getName}%s" + // Filter cross links and add a newline + (Seq("sed", "-e", """s/@ref://g;s/@@.*//g""", f.toString) #> dest).! + // Add newline at the end of the file so as to make pandoc happy + ("echo" #>> dest).! + ("echo \\pagebreak" #>> dest).! + dest + } -paradoxProperties ++= Map( - "github.base_url" -> "https://github.com/locationtech/rasterframes", - "version" -> version.value, - "scaladoc.org.apache.spark.sql.gt" -> "http://rasterframes.io/latest" - //"scaladoc.geotrellis.base_url" -> "https://geotrellis.github.io/scaladocs/latest", - // "snip.pyexamples.base_dir" -> (baseDirectory.value + "/../pyrasterframes/python/test/examples") -) -paradoxTheme := Some(builtinParadoxTheme("generic")) -//paradoxTheme / sourceDirectory := sourceDirectory.value / "main" / "paradox" / "_template" + val output = target.value / pdfFileName.value -Compile / doc / scalacOptions++= Seq( "-J-Xmx6G", "-no-link-warnings") + val header = (Compile / sourceDirectory).value / "latex" / "header.latex" -Tut / run / fork := true + val args = "pandoc" :: + "--from=markdown+pipe_tables" :: + "--to=pdf" :: + "-t" :: "latex" :: + "-s" :: + "--toc" :: + "-V" :: "title:RasterFrames Users' Manual" :: + "-V" :: "author:Astraea, Inc." :: + "-V" :: "geometry:margin=0.75in" :: + "-V" :: "papersize:letter" :: + "--include-in-header" :: header.toString :: + "-o" :: output.toString :: + prepro.map(_.toString).toList -Tut / run / javaOptions := Seq("-Xmx8G", "-Dspark.ui.enabled=false") + log.info(s"Running: ${args.mkString(" ")}") + Process(args, base).! -Compile / paradox := (Compile / paradox).dependsOn(tutQuick).value -Compile / paradox / sourceDirectory := tutTargetDirectory.value -makeSite := makeSite.dependsOn(Compile / unidoc).dependsOn(Compile / paradox).value + log.info("Wrote: " + output) + + output +} + +makePDF := makePDF.dependsOn(Compile / paradox).value diff --git a/docs/documentation-principles.md b/docs/documentation-principles.md new file mode 100644 index 000000000..3f626ddb8 --- /dev/null +++ b/docs/documentation-principles.md @@ -0,0 +1,98 @@ +# Documentation Principles + +This document outlines some concrete considerations for the planned rewrite of the _RasterFrames Users' Manual_. +See [`docs/README.md`](https://github.com/locationtech/rasterframes/blob/develop/docs/README.md) for technical details on the mechanics of building the documentation. + +## Title + +The project is "RasterFrames". Documentation shall use the name "RasterFrames". The RasterFrames runtime is deployed in two forms: `rasterframes` (for Scala/Java/SQL-only), and `pyrasterframes` (Python). But the user should know and think of the project as one thing: RasterFrames. + +## Format + +The documentation shall be rendered in Markdown (`.md`) and Python Markdown (`.pymd`). The source of this documentation is committed in the same project as the code, in `pyrasterframes/src/main/python/docs`. Additional details on processing the docs can be found [here](https://github.com/locationtech/rasterframes/tree/develop/pyrasterframes#running-python-markdown-sources). + +Filenames shall be in skewer case; lower case with dashes ('-') separating words. For example, `foo-in-bar-with-baz.md`. + +## Target Audience + +The target audience for the _RasterFrames User's Manual_ is the intermediate data scientist or developer, already adept at either Python or Scala. Eventually this should be expanded to include SQL adepts. This user may or may not be an expert at EO data usage, so attention to jargon, undefined terms, new concepts etc. should be kept in mind, making use of authoritative external resources to fill in knowledge. + +> Enumerate concepts they are aware of, including: +> * Scene discretization +> * Temporal revisit rate +> * Spatial resolution +> * Spatial extent + +The user may or may not be familiar with Apache Spark, so they should also be guided in filling in minimum requisite knowledge. At a minimum we have to explain what a `SparkSession` is, and that we have to configure it; note the difference between an "action" and "transformation"; what a "collect" action is, and the consequences if the data is large; awareness of partitioning. + +There's also an opportunity to emphasize the scalability benefits over, say, a rasterio/Pandas-only solution (but that we interop with them too). + +The users' goals with EO data are could be from a number of different perspectives: + +* Creating map layers +* Statistical analysis +* Machine learning +* Change detection +* Chip preparation + +While the emphasis of the documentation should be on the core capabilities (and flexibility therein) of RasterFrames, a nod toward these various needs in the examples shown can go a long way in helping the user understand appropriateness of the library to their problem domain. + +## Pedagogical Technique + +The documentation shall emphasize the use of executable code examples, with interspersed prose to explain them. The RasterFrames tooling supports the `.pymd` (Python Markdown) format whereby delimited code blocks are evaluated at build time in order to include output/results. The project currently uses ['Pweave'](http://mpastell.com/pweave/chunks.html) to do this (this may change, but `.pymd` will remain the source format). `Pweave` also has the ability to convert `.pymd` to Jupyter Notebooks, which may serve useful. Through this process we can be assured that any examples shown are code the user can copy into their own projects. + +Visuals are always helpful, but even more so when there's a lot of code involved that needs continual contextualization and explanation. `Pweave` supports rendering of `matplotlib` charts/images, a capability we should make use of. Furthermore, where beneficial, we should create diagrams or other visuals to help express concepts and relationships. + +This "code-first" focus is admittedly in tension with the competing need to explain some of the more abstract aspects of distributed computing necessary for advanced monitoring, profiling, optimization, and deployment. We should evolve the documentation over the long term to address some of these needs, but in the near term the focus should be on the core conceptual model necessary for understanding tile processing. Diagrams can be helpful here. + +## Polyglot Considerations + +In terms of implementation, RasterFrames is a Scala project first. All algorithmic, data modeling, and heavy lifting, etc. are implemented in Scala. + +However, due to user base preferences, RasterFrames is primarily _deployed_ through Python. As such, documentation, examples, etc. should first be implemented in Python. + +Secondarily to that, SQL should be used to highlight the analyst-friendly expression of the functionality in SQL. At least a handful of examples in a dedicated SQL page would go far in showing the cross-language support. + +Thirdly, Scala developers should be encouraged to use the platform, clearly stating that the APIs are on equal footing, using consistent naming conventions, etc. and that most examples should translate almost one-to-one. + +In the long term I'd love to see Python, Scala, and SQL all treated in equal footing, with examples expressed in all languages, but that's a tall order this early in the project development. + +## User Journey + +As noted in the _Target Audience_ section, the documentation needs to guide the user through process from curiosity around EO data to scalable processing of it. Within the first section or so the user should see an example that reads an image and does something somewhat compelling with it, noting that the same code will work on a laptop with a small amount of imagery, as well as on 100s of computers with TB (or more) of imagery. Problems "solved in the small" can be grown to "global scale". + +With a "journey" focus, concepts and capabilities are introduced incrementally, building upon previous examples and concepts, adding more complexity as it develops. Once the fundamentals are covered, we then move into the examples that are closer to use-cases or cookbook entries. + +The preliminary outline is a follows, but is open for refinement, rethinking, etc. + +1. Description +2. Architecture +3. Getting Started + * `pyspark` shell + * Jupyter Notebook + * Standalone Python Script +4. Raster Data I/O + * Reading Raster Data + * Writing Raster Data +5. Spatial Relations +6. Raster Processing + * Local Algebra + * “NoData” Handling + * Aggregation + * Time Series + * Spark ML Support + * Pandas and NumPy Interoperability +7. Cookbook Examples +8. Extended Examples / Case Studies +9. Function Reference + +## Hiccups + +During the documentation process we are likely to run into problems whereby the goal of the writer is inhibited by a bug or capability gap in the code. We should use this opportunity to improve the library to provide the optimal user experience before resorting to workarounds or hacks or addition of what might seem to be arbitrary complexity. + +## Testing + +To "be all that we can be", testing the documentation against a new user is a boon. It may be hard to capture volunteers to do this, but we should consider enlisting interns and friends of the company to go through the documentation and give feedback on where gaps exist. + + + diff --git a/docs/src/main/latex/header.latex b/docs/src/main/latex/header.latex new file mode 100644 index 000000000..50a53b5da --- /dev/null +++ b/docs/src/main/latex/header.latex @@ -0,0 +1,9 @@ +\DeclareUnicodeCharacter{2218}{$\circ$} +\DeclareUnicodeCharacter{2714}{$\checkmark$} +\DeclareUnicodeCharacter{21A9}{$\newline$} +\hypersetup{ + colorlinks=true, + linkcolor=blue, + allbordercolors={0 0 0}, + pdfborderstyle={/S/U/W 1} +} \ No newline at end of file diff --git a/docs/src/main/tut/CNAME b/docs/src/main/paradox/CNAME similarity index 100% rename from docs/src/main/tut/CNAME rename to docs/src/main/paradox/CNAME diff --git a/docs/src/main/paradox/RasterFramePipeline.png b/docs/src/main/paradox/RasterFramePipeline.png new file mode 100644 index 000000000..26900b8cf Binary files /dev/null and b/docs/src/main/paradox/RasterFramePipeline.png differ diff --git a/docs/src/main/tut/RasterFramePipeline.svg b/docs/src/main/paradox/RasterFramePipeline.svg similarity index 100% rename from docs/src/main/tut/RasterFramePipeline.svg rename to docs/src/main/paradox/RasterFramePipeline.svg diff --git a/docs/src/main/paradox/_template/page.st b/docs/src/main/paradox/_template/page.st index c264d9e7c..2a32dd170 100644 --- a/docs/src/main/paradox/_template/page.st +++ b/docs/src/main/paradox/_template/page.st @@ -31,6 +31,8 @@ .md-left { float: left; } .md-right { float: right; } .md-clear { clear: both; } + table { font-size: 80%; } + code { font-size: 0.75em !important; } @@ -132,7 +134,7 @@
                          - © $page.properties.("date.year")$ + Copyright © $page.properties.("date.year")$ Astraea, Inc.
                          diff --git a/docs/src/main/tut/release-notes.md b/docs/src/main/paradox/release-notes.md similarity index 73% rename from docs/src/main/tut/release-notes.md rename to docs/src/main/paradox/release-notes.md index c043ac772..81fe395b0 100644 --- a/docs/src/main/tut/release-notes.md +++ b/docs/src/main/paradox/release-notes.md @@ -4,17 +4,45 @@ ### 0.8.0 -* Added new tile functions `round`, `log`, `log10`, `log2`, `log1p`, `exp`, `exp10`, `exp2`, `expm1`, `resample`, `resample`. +* Super-duper new Python-centric [RasterFrames Users' Manual](https://rasterframes.io/)! +* Upgraded to the following core dependencies: Spark 2.3.3, GeoTrellis 2.3.0, GeoMesa 2.2.1, JTS 1.16.0. +* Build `pyrasterframes` binary distribution for pip installation. +* Added support for rendering RasterFrame types in IPython/Jupyter. +* Added new tile functions `rf_round`, `rf_abs`, `rf_log`, `rf_log10`, `rf_log2`, `rf_log1p`, `rf_exp`, `rf_exp10`, `rf_exp2`, `rf_expm1`, `rf_resample`. +* Support Python-side Tile User-Defined Type backed by [numpy](https://www.numpy.org/) `ndarray` or `ma.MaskedArray`. +* Support Python-side [Shapely](https://pypi.org/project/Shapely/) geometry User-Defined Type. +* SQL API support for `rf_assemble_tile` and `rf_array_to_tile`. * Introduced at the source level the concept of a `RasterSource` and `RasterRef`, enabling lazy/delayed read of sub-scene tiles. -* _Deprecation_: Tile column functions (in `RasterFunctions`) and SQL registered names have all been renamed to follow `snake_case` conventions, matching SQL and Python. A temporary compatibility shim is included so that code built against 0.7.1 and earlier still work. These will be marked as deprecated. * Added `withKryoSerialization` extension methods on `SparkSession.Builder` and `SparkConf`. +* Added `rf_render_matrix` debugging function. +* Added `RasterFrameLayer.withExtent` extension method. +* Added `SinglebandGeoTiff.toDF` extension method. +* Added `DataFrame.rasterJoin` extension method for merging two dataframes with tiles in disparate CRSs. +* Added `rf_crs` for `ProjectedRasterTile` columns. +* Added `st_extent` (for `Geometry` types) and `rf_extent` (for `ProjectedRasterTile` and `RasterSource` columns). +* Added `st_geometry` (for `Extent` types) and `rf_geometry` (for `ProjectedRasterTile` and `RasterSource` columns). +* Reworked build scripts for RasterFrames Jupyter Notebook. +* _Breaking_: The type `RasterFrame` renamed `RasterFrameLayer` to be reflect its intended purpose. +* _Breaking_: All `asRF` methods renamed to `asLayer`. +* _Breaking_: Root package changed from `org.locationtech.rasterframes` to `org.locationtech.rasterframes`. +* _Breaking_: Removed `envelope`, in lieu of `st_extent`, `rf_extent` or `st_envelope` +* _Breaking_: Renamed `rf_extent_geometry` to `st_geometry` +* _Breaking_: Renamed `rf_tile_dimensions` to `rf_dimensions` +* _Breaking_: Renamed `rf_reproject_geometry` to `st_reproject` +* _Breaking_: With the upgrade to JTS 1.16.0, all imports of `com.vividsolutions.jts` need to be changed to `org.locationtech.jts`. +* _Deprecation_: Tile column functions (in `RasterFunctions`) and SQL registered names have all been renamed to follow `snake_case` conventions, with an `rf_` prefix, matching SQL and Python. A temporary compatibility shim is included so that code built against 0.7.1 and earlier still work. These will be marked as deprecated. * _Breaking_: In Scala and SQL, `..._scalar` functions (e.g. `local_add_scalar`) have been removed. Non-scalar forms now dynamically detect type of right hand side. -* _Breaking_: `tileToArray` has been replaced with `tile_to_array_double` and `tile_to_array_int`. -* Added `render_matrix` debugging function. -* _Breaking_: renamed `agg_histogram` to `agg_approx_histogram`, `local_agg_stats` to `agg_local_stats`, `local_agg_max` to `agg_local_max`, `local_agg_min` to `agg_local_min`, `local_agg_mean` to `agg_local_mean`, `local_agg_data_cells` to `agg_local_data_cells`, `local_agg_no_data_cells` to `agg_local_no_data_cells`. -* _Breaking_: `CellHistogram` no longer carries along approximate statistics, due to confusing behavior. Use `agg_stats` instead. +* _Breaking_: `tileToArray` has been replaced with `_tile_to_array_double` and `_tile_to_array_int`. +* _Breaking_: Renamed `bounds_geometry` to `rf_extent_geometry`. +* _Breaking_: renamed `agg_histogram` to `rf_agg_approx_histogram`, `local_agg_stats` to `rf_agg_local_stats`, `local_agg_max` to `rf_agg_local_max`, `local_agg_min` to `rf_agg_local_min`, `local_agg_mean` to `rf_agg_local_mean`, `local_agg_data_cells` to `rf_agg_local_data_cells`, `local_agg_no_data_cells` to `rf_agg_local_no_data_cells`. +* _Breaking_: `CellHistogram` no longer carries along approximate statistics, due to confusing behavior. Use `rf_agg_stats` instead. * Introduced `LocalCellStatistics` class to wrap together results from `LocalStatsAggregate`. -* _Breaking_: `TileDimensions` moved from `astraea.spark.rasterframes` to `astraea.spark.rasterframes.model`. +* _Breaking_: `TileDimensions` moved from `astraea.spark.rasterframes` to `org.locationtech.rasterframes.model`. +* _Breaking_: Renamed `RasterFrame.withBounds` to `RasterFrameLayer.withGeometry` for consistency with DataSource schemas. + +#### Known issues + +* [#188](https://github.com/locationtech/rasterframes/issues/188): Error on deserialization of a `Tile` with a `bool` cell type to the Python side; see issue description for work around. ## 0.7.x diff --git a/docs/src/main/tut/apps/geotrellis-ops.md b/docs/src/main/tut/apps/geotrellis-ops.md deleted file mode 100644 index 81a97a3fa..000000000 --- a/docs/src/main/tut/apps/geotrellis-ops.md +++ /dev/null @@ -1,44 +0,0 @@ -# GeoTrellis Operations - -```tut:invisible -import astraea.spark.rasterframes._ -import geotrellis.raster._ -import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ - -implicit val spark = SparkSession.builder(). - withKryoSerialization. - master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") -import spark.implicits._ -val scene = SinglebandGeoTiff("../core/src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128) -``` - - -GeoTrellis provides a [rich set of Map Algebra operations](https://docs.geotrellis.io/en/latest/guide/core-concepts.html#map-algebra) and other tile processing features that can be used with RasterFrames via Spark's UDF support. - -Here's an example creating a UDFs to invoke the `equalize` transformation on each tile in a RasterFrame, and then compute the resulting per-tile mean of it. - -```tut -import geotrellis.raster.equalization._ -val equalizer = udf((t: Tile) => t.equalize()) -val equalized = rf.select(equalizer($"tile") as "equalized") -equalized.select(tile_mean($"equalized") as "equalizedMean").show(5, false) -``` - -Here's an example downsampling a tile and rendering each tile as a matrix of numerical values. - -```tut -val downsample = udf((t: Tile) => t.resample(4, 4)) -val downsampled = rf.where(no_data_cells($"tile") === 0).select(downsample($"tile") as "minime") -downsampled.select(tile_to_array_double($"minime") as "cell_values").limit(2).show(false) -``` - - -```tut:invisible -spark.stop() -``` - diff --git a/docs/src/main/tut/apps/index.md b/docs/src/main/tut/apps/index.md deleted file mode 100644 index c0f7bc28e..000000000 --- a/docs/src/main/tut/apps/index.md +++ /dev/null @@ -1,12 +0,0 @@ -# Applications - -In this section we outline a few use cases for using *RasterFrames* in your own applications. - -@@ toc { depth=2 } - -@@@ index - -* [GeoTrellis Operations](geotrellis-ops.md) -* [Calculating NDVI](ndvi.md) - -@@@ diff --git a/docs/src/main/tut/apps/ndvi.md b/docs/src/main/tut/apps/ndvi.md deleted file mode 100644 index a9fb72a3e..000000000 --- a/docs/src/main/tut/apps/ndvi.md +++ /dev/null @@ -1,61 +0,0 @@ -# Computing NDVI - -```tut:invisible -import astraea.spark.rasterframes._ -import geotrellis.raster._ -import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import geotrellis.spark._ -import geotrellis.spark.io._ -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ - -implicit val spark = SparkSession.builder(). - withKryoSerialization. - master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") -import spark.implicits._ -val scene = SinglebandGeoTiff("../core/src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128).cache() -``` - -Here's an example of computing the Normalized Differential Vegetation Index (NDVI) is a -standardized vegetation index which allows us to generate an image highlighting differences in -relative biomass. - -> “An NDVI is often used worldwide to monitor drought, monitor and predict agricultural production, assist in predicting hazardous fire zones, and map desert encroachment. The NDVI is preferred for global vegetation monitoring because it helps to compensate for changing illumination conditions, surface slope, aspect, and other extraneous factors” (Lillesand. *Remote sensing and image interpretation*. 2004). - -```tut:silent -def redBand = SinglebandGeoTiff("../core/src/test/resources/L8-B4-Elkton-VA.tiff").projectedRaster.toRF("red_band") -def nirBand = SinglebandGeoTiff("../core/src/test/resources/L8-B5-Elkton-VA.tiff").projectedRaster.toRF("nir_band") - -// We use `asRF` to indicate we know the structure still conforms to RasterFrame constraints -val rf = redBand.spatialJoin(nirBand).withColumn("ndvi", normalized_difference($"red_band", $"nir_band")).asRF - -val pr = rf.toRaster($"ndvi", 466, 428) - -val brownToGreen = ColorRamp( - RGBA(166,97,26,255), - RGBA(223,194,125,255), - RGBA(245,245,245,255), - RGBA(128,205,193,255), - RGBA(1,133,113,255) -).stops(128) - -val colors = ColorMap.fromQuantileBreaks(pr.tile.histogramDouble(), brownToGreen) -pr.tile.color(colors).renderPng().write("target/scala-2.11/tut/apps/rf-ndvi.png") -``` - -![](rf-ndvi.png) - -For a georefrenced singleband greyscale image, we could have done this instead: - -```scala -GeoTiff(pr).write("ndvi.tiff") -``` - - -```tut:invisible -spark.stop() -``` - diff --git a/docs/src/main/tut/creating-rasterframes.md b/docs/src/main/tut/creating-rasterframes.md deleted file mode 100644 index 7e7ab2c66..000000000 --- a/docs/src/main/tut/creating-rasterframes.md +++ /dev/null @@ -1,212 +0,0 @@ -# Creating RasterFrames - -## Initialization - -There are a couple of setup steps necessary anytime you want to work with RasterFrames. the first is to import the API symbols into scope: - - - -```tut:silent -import astraea.spark.rasterframes._ -import org.apache.spark.sql._ -``` - - -Next, initialize the `SparkSession`, and call the `withRasterFrames` method on it: - - -```tut:silent -implicit val spark = SparkSession.builder(). - master("local").appName("RasterFrames"). - config("spark.ui.enabled", "false"). - withKryoSerialization. - getOrCreate(). - withRasterFrames -``` - -And, as is standard Spark SQL practice, we import additional DataFrame support: - -```tut:silent -import spark.implicits._ -``` - -```tut:invisible -spark.sparkContext.setLogLevel("ERROR") -``` - -Now we are ready to create a RasterFrame. - -## Reading a GeoTIFF - -The most straightforward way to create a `RasterFrame` is to read a [GeoTIFF](https://en.wikipedia.org/wiki/GeoTIFF) -file using a RasterFrame [`DataSource`](https://spark.apache.org/docs/latest/sql-programming-guide.html#data-sources) -designed for this purpose. [Cloud Optimized GeoTIFF](http://www.cogeo.org/) files are preferred, as they don't require re-tiling and can be read in a more memory-efficient way. - -First add the following import: - - -```tut:silent -import astraea.spark.rasterframes.datasource.geotiff._ -import java.io.File -``` - -(This is what adds the `.geotiff` method to `spark.read` below.) - -Then we use the `DataFrameReader` provided by `spark.read` to read the GeoTIFF: - - -```tut:book -val samplePath = new File("../core/src/test/resources/LC08_RGB_Norfolk_COG.tiff") -val tiffRF = spark.read. - geotiff. - loadRF(samplePath.toURI) -``` - - -Let's inspect the structure of what we get back: - - -```tut -tiffRF.printSchema() -``` - -As reported by Spark, RasterFrames extracts 6 columns from the GeoTIFF we selected. Some of these columns are dependent -on the contents of the source data, and some are are always available. Let's take a look at these in more detail. - -* `spatial_key`: GeoTrellis assigns a `SpatialKey` or a `SpaceTimeKey` to each tile, mapping it to the layer grid from -which it came. If it has a `SpaceTimeKey`, RasterFrames will split it into a `SpatialKey` and a `TemporalKey` (the -latter with column name `temporal_key`). -* `extent`: The bounding box of the tile in the tile's native CRS. -* `metadata`: The TIFF format header tags found in the file. -* `tile` or `tile_n` (where `n` is a band number): For singleband GeoTIFF files, the `tile` column contains the cell -data split into tiles. For multiband tiles, each column with `tile_` prefix contains each of the sources bands, -in the order they were stored. - -See the section [Inspecting a `RasterFrame`](#inspecting-a--code-rasterframe--code-) (below) for more details on accessing the RasterFrame's metadata. - -## Reading a GeoTrellis Layer - -If your imagery is already ingested into a [GeoTrellis layer](https://docs.geotrellis.io/en/latest/guide/spark.html#writing-layers), -you can use the RasterFrames GeoTrellis DataSource. There are two parts to this GeoTrellis Layer support. The first -is the GeoTrellis Catalog DataSource, which lists the GeoTrellis layers available at a URI. The second part is the actual -RasterFrame reader for pulling a layer into a RasterFrame. - -Before we show how all of this works we need to have a GeoTrellis layer to work with. We can create one with the RasterFrame we constructed above. - - -```tut:silent -import astraea.spark.rasterframes.datasource.geotrellis._ -import java.nio.file.Files - -val base = Files.createTempDirectory("rf-").toUri -val layer = Layer(base, "sample", 0) -tiffRF.write.geotrellis.asLayer(layer).save() -``` - -Now we can point our catalog reader at the base directory and see what was saved: - -```tut -val cat = spark.read.geotrellisCatalog(base) -cat.printSchema -cat.show() -``` - -As you can see, there's a lot of information stored in each row of the catalog. Most of this is associated with how the -layer is discretized. However, there may be other application specific metadata serialized with a layer that can be use -to filter the catalog entries or select a specific one. But for now, we're just going to load a RasterFrame in from the -catalog using a convenience function. - -```tut -val firstLayer = cat.select(geotrellis_layer).first -val rfAgain = spark.read.geotrellis.loadRF(firstLayer) -rfAgain.show() -``` - -If you already know the `LayerId` of what you're wanting to read, you can bypass working with the catalog: - -```tut -val anotherRF = spark.read.geotrellis.loadRF(layer) -``` - -## Writing a GeoTrellis Layer - -See @ref:[Exporting to a GeoTrellis Layer](creating-rasterframes.md#exporting-to-a-geotrellis-layer). - -## Using GeoTrellis APIs - -If you are used to working directly with the GeoTrellis APIs, there are a number of additional ways to create a `RasterFrame`, as enumerated in the sections below. - -First, some more `import`s: - -```tut:silent -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import geotrellis.spark.io._ -``` - - -### From `ProjectedExtent` - -The simplest mechanism for getting a RasterFrame is to use the `toRF(tileCols, tileRows)` extension method on `ProjectedRaster`. - - -```tut -val scene = SinglebandGeoTiff("../core/src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128) -rf.show(5, false) -``` - - -### From `TileLayerRDD` - -Another option is to use a GeoTrellis [`LayerReader`](https://docs.geotrellis.io/en/latest/guide/tile-backends.html), -to get a `TileLayerRDD` for which there's also a `toRF` extension method. - - -```scala -import geotrellis.spark._ -val tiledLayer: TileLayerRDD[SpatialKey] = ??? -val rf = tiledLayer.toRF -``` - - -## Inspecting a `RasterFrame` - -`RasterFrame` has a number of methods providing access to metadata about the contents of the RasterFrame. - -### Tile Column Names - - -```tut:book -rf.tileColumns.map(_.toString) -``` - - -### Spatial Key Column Name - - -```tut:book -rf.spatialKeyColumn.toString -``` - -### Temporal Key Column - -Returns an `Option[Column]` since not all RasterFrames have an explicit temporal dimension. - -```tut:book -rf.temporalKeyColumn.map(_.toString) -``` - -### Tile Layer Metadata - -The Tile Layer Metadata defines how the spatial/spatiotemporal domain is discretized into tiles, and what the key bounds are. - -```tut -import spray.json._ -// NB: The `fold` is required because an `Either` is returned, depending on the key type. -rf.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint -``` - -```tut:invisible -spark.stop() -``` - diff --git a/docs/src/main/tut/exporting-rasterframes.md b/docs/src/main/tut/exporting-rasterframes.md deleted file mode 100644 index 2015943f5..000000000 --- a/docs/src/main/tut/exporting-rasterframes.md +++ /dev/null @@ -1,190 +0,0 @@ -# Exporting RasterFrames - -```tut:invisible -import astraea.spark.rasterframes._ -import geotrellis.spark._ -import geotrellis.raster._ -import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ - -implicit val spark = SparkSession.builder().config("spark.ui.enabled", "false"). - withKryoSerialization. - master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") -import spark.implicits._ -val scene = SinglebandGeoTiff("../core/src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128).cache() -``` - -While the goal of RasterFrames is to make it as easy as possible to do your geospatial analysis with a single -construct, it is helpful to be able to transform it into other representations for various use cases. - -## Converting to Array - -The cell values within a `Tile` are encoded internally as an array. There may be use cases -where the additional context provided by the `Tile` construct is no longer needed and one would -prefer to work with the underlying array data. - -The @scaladoc[`tile_to_array_int`][tile_to_array_int] column function requires a type parameter to indicate the array element -type you would like used. The following types may be used: `Int`, `Double`, `Byte`, `Short`, `Float` - -```tut -val withArrays = rf.withColumn("tileData", tile_to_array_int($"tile")).drop("tile") -withArrays.show(5, 40) -``` - -You can convert the data back to an array, but you have to specify the target tile dimensions. - -```tut -val tileBack = withArrays.withColumn("tileAgain", arrayToTile($"tileData", 128, 128)) -tileBack.drop("tileData").show(5, 40) -``` - -Note that the created tile will not have a `NoData` value associated with it. Here's how you can do that: - -```tut -val tileBackAgain = withArrays.withColumn("tileAgain", withNoData(arrayToTile($"tileData", 128, 128), 3)) -tileBackAgain.drop("tileData").show(5, 50) -``` - -## Writing to Parquet - -It is often useful to write Spark results in a form that is easily reloaded for subsequent analysis. -The [Parquet](https://parquet.apache.org/)columnar storage format, native to Spark, is ideal for this. RasterFrames -work just like any other DataFrame in this scenario as long as @scaladoc[`rfInit`][rfInit] is called to register -the imagery types. - - -Let's assume we have a RasterFrame we've done some fancy processing on: - -```tut:silent -import geotrellis.raster.equalization._ -val equalizer = udf((t: Tile) => t.equalize()) -val equalized = rf.withColumn("equalized", equalizer($"tile")).asRF -``` - -```tut -equalized.printSchema -equalized.select(aggStats($"tile")).show(false) -equalized.select(aggStats($"equalized")).show(false) -``` - -We write it out just like any other DataFrame, including the ability to specify partitioning: - -```tut:silent -val filePath = "/tmp/equalized.parquet" -equalized.select("*", "spatial_key.*").write.partitionBy("col", "row").mode(SaveMode.Overwrite).parquet(filePath) -``` - -Let's confirm partitioning happened as expected: - -```tut -import java.io.File -new File(filePath).list.filter(f => !f.contains("_")) -``` - -Now we can load the data back in and check it out: - -```tut:silent -val rf2 = spark.read.parquet(filePath) -``` - -```tut -rf2.printSchema -equalized.select(aggStats($"tile")).show(false) -equalized.select(aggStats($"equalized")).show(false) -``` - - -## Exporting a Raster - -For the purposes of debugging, the RasterFrame tiles can be reassembled back into a raster for viewing. However, -keep in mind that this will download all the data to the driver, and reassemble it in-memory. So it's not appropriate -for very large coverages. - -Here's how one might render the image to a georeferenced GeoTIFF file: - -```tut:silent -import geotrellis.raster.io.geotiff.GeoTiff -val image = equalized.toRaster($"equalized", 774, 500) -GeoTiff(image).write("target/scala-2.11/tut/rf-raster.tiff") -``` - -[*Download GeoTIFF*](rf-raster.tiff) - -Here's how one might render a raster frame to a false color PNG file: - -```tut:silent -val colors = ColorMap.fromQuantileBreaks(image.tile.histogram, ColorRamps.BlueToOrange) -image.tile.color(colors).renderPng().write("target/scala-2.11/tut/rf-raster.png") -``` - -![](rf-raster.png) - -## Exporting to a GeoTrellis Layer - -For future analysis it is helpful to persist a RasterFrame as a [GeoTrellis layer](http://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html). - -First, convert the RasterFrame into a TileLayerRDD. The return type is an Either; -the `left` side is for spatial-only keyed data - -```tut:book -val tlRDD = equalized.toTileLayerRDD($"equalized").left.get -``` - -Then create a GeoTrellis layer writer: - -```tut:silent -import java.nio.file.Files -import spray.json._ -import DefaultJsonProtocol._ -import geotrellis.spark.io._ -val p = Files.createTempDirectory("gt-store") -val writer: LayerWriter[LayerId] = LayerWriter(p.toUri) - -val layerId = LayerId("equalized", 0) -writer.write(layerId, tlRDD, index.ZCurveKeyIndexMethod) -``` - -Take a look at the metadata in JSON format: -```tut -AttributeStore(p.toUri).readMetadata[JsValue](layerId).prettyPrint -``` - -## Converting to `RDD` and `TileLayerRDD` - -Since a `RasterFrame` is just a `DataFrame` with extra metadata, the method -@scaladoc[`DataFrame.rdd`][rdd] is available for simple conversion back to `RDD` space. The type returned -by `.rdd` is dependent upon how you select it. - - -```tut -import scala.reflect.runtime.universe._ -def showType[T: TypeTag](t: T) = println(implicitly[TypeTag[T]].tpe.toString) - -showType(rf.rdd) - -showType(rf.select(rf.spatialKeyColumn, $"tile".as[Tile]).rdd) - -showType(rf.select(rf.spatialKeyColumn, $"tile").as[(SpatialKey, Tile)].rdd) -``` - -If your goal convert a single tile column with its spatial key back to a `TileLayerRDD[K]`, then there's an additional -extension method on `RasterFrame` called [`toTileLayerRDD`][toTileLayerRDD], which preserves the tile layer metadata, -enhancing interoperation with GeoTrellis RDD extension methods. - -```tut -showType(rf.toTileLayerRDD($"tile".as[Tile])) -``` - -```tut:invisible -spark.stop() -``` - -[rfInit]: astraea.spark.rasterframes.package#rfInit%28SQLContext%29:Unit -[rdd]: org.apache.spark.sql.Dataset#frdd:org.apache.spark.rdd.RDD[T] -[toTileLayerRDD]: astraea.spark.rasterframes.RasterFrameMethods#toTileLayerRDD%28tileCol:RasterFrameMethods.this.TileColumn%29:Either[geotrellis.spark.TileLayerRDD[geotrellis.spark.SpatialKey],geotrellis.spark.TileLayerRDD[geotrellis.spark.SpaceTimeKey]] -[tile_to_array_int]: astraea.spark.rasterframes.ColumnFunctions#tile_to_array_int - diff --git a/docs/src/main/tut/getting-started.md b/docs/src/main/tut/getting-started.md deleted file mode 100644 index 413131e84..000000000 --- a/docs/src/main/tut/getting-started.md +++ /dev/null @@ -1,49 +0,0 @@ -# Getting Started - -## Quick Start - -RasterFrames is available in in a Jupyter Notebook Docker container for quick experimentation. - -1. Install [Docker](https://www.docker.com/get-docker) for your OS flavor. -2. Run - `docker run -it --rm -p 8888:8888 -p 4040-4044:4040-4044 s22s/rasterframes-notebooks` - -Additional instructions can be found [here](https://github.com/locationtech/rasterframes/blob/develop/deployment/README.md). - -## General Setup - -*RasterFrames* is published via [Maven Central](https://search.maven.org/search?q=g:org.locationtech.rasterframes) (click link to see latest versions). - -To use RasterFrames, add the following library dependencies: - -@@dependency[sbt,Maven,Gradle] { - group="org.locationtech.rasterframes" - artifact="rasterframes_2.11" - version="$version$" -} - -@@dependency[sbt,Maven,Gradle] { - group="org.locationtech.rasterframes" - artifact="rasterframes-datasource_2.11" - version="$version$" -} - -Optional: - -@@dependency[sbt,Maven,Gradle] { - group="org.locationtech.rasterframes" - artifact="rasterframes-experimental_2.11" - version="$version$" -} - -It assumes that SparkSQL 2.3.x is available in the runtime classpath. Here's how to add it explicitly: - -@@dependency[sbt,Maven,Gradle] { - group="org.apache.spark" - artifact="spark-sql" - version="2.3.2" -} - -@@@ note -Most of the following examples are shown using the Spark DataFrames API. However, many could also be rewritten to use the Spark SQL API instead. We hope to add more examples in that form in the future. -@@@ diff --git a/docs/src/main/tut/index.md b/docs/src/main/tut/index.md deleted file mode 100644 index 7351536be..000000000 --- a/docs/src/main/tut/index.md +++ /dev/null @@ -1,51 +0,0 @@ -# RasterFrames - -RasterFrames® brings together Earth-observing (EO) data analysis, big data computing, and DataFrame-based data science. The recent explosion of EO data from public and private satellite operators presents both a huge opportunity as well as a challenge to the data analysis community. It is _Big Data_ in the truest sense, and its footprint is rapidly getting bigger. - -RasterFrames provides a DataFrame-centric view over arbitrary EO data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of Spark ML algorithms. By using DataFrames as the core cognitive and compute data model, it is able to deliver these features in a form that is accessible to general analysts while handling the rapidly growing data footprint. - -To learn more, please see the @ref:[Getting Started](getting-started.md) section of this manual. - -The source code can be found on GitHub at [locationtech/rasterframes](https://github.com/locationtech/rasterframes). - - - -
                          - -@@@ div { .md-left} - -## Detailed Contents - -@@ toc { depth=2 } - -@@@ - -@@@ div { .md-right } - -## Related Links - -* [Gitter Channel](https://gitter.im/s22s/raster-frames)  [![Join the chat at https://gitter.im/s22s/raster-frames](https://badges.gitter.im/s22s/raster-frames.svg)](https://gitter.im/s22s/raster-frames?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -* [API Documentation](latest/api/index.html) -* [GitHub Repository](https://github.com/locationtech/rasterframes) -* [Astraea, Inc.](http://www.astraea.earth/) (the company behind RasterFrames) - -@@@ - -@@@ div { .md-clear } - -  - -@@@ - -@@@ index -* [Getting Started](getting-started.md) -* [PyRasterFrames](pyrasterframes.md) -* [Creating RasterFrames](creating-rasterframes.md) -* [Spatial Queries](spatial-queries.md) -* [Applications](apps/index.md) -* [Machine Learning](ml/index.md) -* [Exporting Rasterframes](exporting-rasterframes.md) -* [UDF Reference](reference.md) -* [Release Notes](release-notes.md) -@@@ - diff --git a/docs/src/main/tut/ml/L8-RGB-VA.png b/docs/src/main/tut/ml/L8-RGB-VA.png deleted file mode 100644 index cb41d2742..000000000 Binary files a/docs/src/main/tut/ml/L8-RGB-VA.png and /dev/null differ diff --git a/docs/src/main/tut/ml/classification.md b/docs/src/main/tut/ml/classification.md deleted file mode 100644 index 55c8ced98..000000000 --- a/docs/src/main/tut/ml/classification.md +++ /dev/null @@ -1,229 +0,0 @@ -# Classification - -In this example we will do some simple cell classification based on multiband imagery and a -target/label raster. As a part of the process we'll explore the cross-validation support in -SparkML. - -## Setup - -First some setup: - -```tut:silent -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.ml.{NoDataFilter, TileExploder} -import geotrellis.raster._ -import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.classification.DecisionTreeClassifier -import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator -import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.ml.tuning.{CrossValidator, ParamGridBuilder} -import org.apache.spark.sql._ - -implicit val spark = SparkSession.builder(). - withKryoSerialization. - master("local[*]").appName(getClass.getName).getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") - -import spark.implicits._ - -// Utility for reading imagery from our test data set -def readTiff(name: String): SinglebandGeoTiff = SinglebandGeoTiff(s"../core/src/test/resources/$name") -``` - -## Loading Data - -The first step is to load multiple bands of imagery and construct a single RasterFrame from them. -To do this we: - -1. Identify the GeoTIFF filename. -2. Read the TIFF raster -3. Convert to a raster frame of `tileSize` sized tiles, with an appropriate column name -4. Use the RasterFrames `spatialJoin` function to create a new RasterFrame with a column for each band - - -```tut:silent -val filenamePattern = "L8-%s-Elkton-VA.tiff" -val bandNumbers = 2 to 7 -val bandColNames = bandNumbers.map(b ⇒ s"band_$b").toArray -val tileSize = 10 - -// For each identified band, load the associated image file -val joinedRF = bandNumbers. - map { b ⇒ (b, filenamePattern.format("B" + b)) }. - map { case (b, f) ⇒ (b, readTiff(f)) }. - map { case (b, t) ⇒ t.projectedRaster.toRF(tileSize, tileSize, s"band_$b") }. - reduce(_ spatialJoin _) -``` - -We should see a single `spatial_key` column along with 6 columns of tiles. - -```tut -joinedRF.printSchema() -``` - -Similarly pull we pull in the target label data. When load the target label raster we have -to convert the cell type to `Double` to meet expectations of SparkML. - -```tut:silent -val targetCol = "target" - -val target = readTiff(filenamePattern.format("Labels")). - mapTile(_.convert(DoubleConstantNoDataCellType)). - projectedRaster. - toRF(tileSize, tileSize, targetCol) -``` - -Take a peek at what kind of label data we have to work with. - -```tut -target.select(agg_stats(target(targetCol))).show -``` - -Join the target label RasterFrame with the band tiles to create our analytics base table - -```tut:silent -val abt = joinedRF.spatialJoin(target) - -``` - -## ML Pipeline - -The data preparation modeling pipeline is next. SparkML requires that each observation be in -its own row, and those observations be packed into a single `Vector` type. The first step is -to "explode" the tiles into a single row per cell/pixel. Then we filter out any rows that -have `NoData` values (which will cause an error during training). Finally we use the -SparkML `VectorAssembler` to create that `Vector`. - -```tut:silent -val exploder = new TileExploder() - -val noDataFilter = new NoDataFilter(). - setInputCols(bandColNames :+ targetCol) - -val assembler = new VectorAssembler(). - setInputCols(bandColNames). - setOutputCol("features") -``` - -We are going to use a decision tree for classification. You can swap out one of the other multi-class -classification algorithms if you like. With the algorithm selected we can assemble our modeling pipeline. - -```tut:silent -val classifier = new DecisionTreeClassifier(). - setLabelCol(targetCol). - setFeaturesCol(assembler.getOutputCol) - -val pipeline = new Pipeline(). - setStages(Array(exploder, noDataFilter, assembler, classifier)) -``` - -## Cross Validation - -To extend the sophistication of the example we are going to use the SparkML support for -cross-validation and hyper-parameter tuning. The first step is to configure how we're -going to evaluate our model's performance. Then we define the hyperparmeter(s) we're going to -vary and evaluate. Finally we configure the cross validator. - -```tut:silent -val evaluator = new MulticlassClassificationEvaluator(). - setLabelCol(targetCol). - setPredictionCol("prediction"). - setMetricName("accuracy") - -val paramGrid = new ParamGridBuilder(). - addGrid(classifier.maxDepth, Array(2, 3, 4)). - build() - -val trainer = new CrossValidator(). - setEstimator(pipeline). - setEvaluator(evaluator). - setEstimatorParamMaps(paramGrid). - setNumFolds(4) -``` - -Push the "go button": - -```tut -val model = trainer.fit(abt) -``` - -## Model Evaluation - -To view the model's performance we format the `paramGrid` settings used for each model and -render the parameter/performance association. - -```tut:silent -val metrics = model.getEstimatorParamMaps. - map(_.toSeq.map(p ⇒ s"${p.param.name} = ${p.value}")). - map(_.mkString(", ")). - zip(model.avgMetrics) -``` -```tut -metrics.toSeq.toDF("params", "metric").show(false) -``` - -Finally, we score the original data set (including the cells without target values) and -add up class membership results. - -```tut -val scored = model.bestModel.transform(joinedRF) - -scored.groupBy($"prediction" as "class").count().show -``` - -## Visualizing Results - -The predictions are in a DataFrame with each row representing a separate pixel. -To assemble a raster to visualize the class assignments, we have to go through a -multi-stage process to get the data back in tile form, and from there to combined -raster form. - -First, we get the DataFrame back into RasterFrame form: - -```tut:silent -val tlm = joinedRF.tileLayerMetadata.left.get - -val retiled = scored.groupBy($"spatial_key").agg( - assemble_tile( - $"column_index", $"row_index", $"prediction", - tlm.tileCols, tlm.tileRows, ByteConstantNoDataCellType - ) -) - -val rf = retiled.asRF($"spatial_key", tlm) -``` - -To render our visualization, we convert to a raster first, and then use an -`IndexedColorMap` to assign each discrete class a different color, and finally -rendering to a PNG file. - -```tut:silent -val raster = rf.toRaster($"prediction", 186, 169) - -val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 until 3).toArray) -) - -raster.tile.renderPng(clusterColors).write("target/scala-2.11/tut/ml/classified.png") -``` - -| Color Composite | Target Labels | Class Assignments | -| ------------------ | ---------------------- | ------------------- | -| ![](L8-RGB-VA.png) | ![](target-labels.png) | ![](classified.png) | - - -```tut:invisible -val raster = SinglebandGeoTiff("../core/src/test/resources/L8-Labels-Elkton-VA.tiff").raster - -val k = raster.tile.findMinMax._2 - -val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 to k).toArray) -) - -raster.tile.renderPng(clusterColors).write("target/scala-2.11/tut/ml/target-labels.png") - -spark.stop() -``` diff --git a/docs/src/main/tut/ml/clustering.md b/docs/src/main/tut/ml/clustering.md deleted file mode 100644 index a3e803ace..000000000 --- a/docs/src/main/tut/ml/clustering.md +++ /dev/null @@ -1,152 +0,0 @@ -# Clustering - -In this example we will do some simple cell clustering based on multiband imagery. - -## Setup - -First some setup: - -```tut:silent -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.ml.TileExploder -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import geotrellis.raster._ -import geotrellis.raster.render._ -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.clustering.{KMeans, KMeansModel} -import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.sql._ - -// Utility for reading imagery from our test data set -def readTiff(name: String): SinglebandGeoTiff = SinglebandGeoTiff(s"../core/src/test/resources/$name") - -implicit val spark = SparkSession.builder(). - withKryoSerialization. - master("local[*]").appName(getClass.getName).getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") - -import spark.implicits._ -``` - -## Loading Data - -The first step is to load multiple bands of imagery and construct a single RasterFrame from them. - -```tut:silent -val filenamePattern = "L8-B%d-Elkton-VA.tiff" -val bandNumbers = 1 to 4 -val bandColNames = bandNumbers.map(b ⇒ s"band_$b").toArray - -// For each identified band, load the associated image file, convert to a RasterFrame, and join -val joinedRF = bandNumbers. - map { b ⇒ (b, filenamePattern.format(b)) }. - map { case (b, f) ⇒ (b, readTiff(f)) }. - map { case (b, t) ⇒ t.projectedRaster.toRF(s"band_$b") }. - reduce(_ spatialJoin _) -``` - -We should see a single `spatial_key` column along with 4 columns of tiles. - -```tut -joinedRF.printSchema() -``` - -## ML Pipeline - -SparkML requires that each observation be in its own row, and those -observations be packed into a single `Vector`. The first step is to -"explode" the tiles into a single row per cell/pixel. - -```tut:silent -val exploder = new TileExploder() -``` - -To "vectorize" the the band columns, as required by SparkML, we use the SparkML -`VectorAssembler`. We then configure our algorithm, create the transformation pipeline, -and train our model. (Note: the selected value of *K* below is arbitrary.) - -```tut:silent -val assembler = new VectorAssembler(). - setInputCols(bandColNames). - setOutputCol("features") - -// Configure our clustering algorithm -val k = 5 -val kmeans = new KMeans().setK(k) - -// Combine the two stages -val pipeline = new Pipeline().setStages(Array(exploder, assembler, kmeans)) - -// Compute clusters -val model = pipeline.fit(joinedRF) -``` - -## Model Evaluation - -At this point the model can be saved off for later use, or used immediately on the same -data we used to compute the model. First we run the data through the model to assign -cluster IDs to each cell. - -```tut -val clustered = model.transform(joinedRF) -clustered.show(8) -``` - -If we want to inspect the model statistics, the SparkML API requires us to go -through this unfortunate contortion: - -```tut:silent -val clusterResults = model.stages.collect{ case km: KMeansModel ⇒ km}.head -``` - -Compute sum of squared distances of points to their nearest center: - -```tut -val metric = clusterResults.computeCost(clustered) -println("Within set sum of squared errors: " + metric) -``` - -## Visualizing Results - -The predictions are in a DataFrame with each row representing a separate pixel. -To assemble a raster to visualize the cluster assignments, we have to go through a -multi-stage process to get the data back in tile form, and from there to combined -raster form. - -First, we get the DataFrame back into RasterFrame form: - -```tut:silent -val tlm = joinedRF.tileLayerMetadata.left.get - -val retiled = clustered.groupBy($"spatial_key").agg( - assemble_tile( - $"column_index", $"row_index", $"prediction", - tlm.tileCols, tlm.tileRows, ByteConstantNoDataCellType - ) -) - -val rf = retiled.asRF($"spatial_key", tlm) -``` - -To render our visualization, we convert to a raster first, and then use an -`IndexedColorMap` to assign each discrete cluster a different color, and finally -rendering to a PNG file. - -```tut:silent -val raster = rf.toRaster($"prediction", 186, 169) - -val clusterColors = IndexedColorMap.fromColorMap( - ColorRamps.Viridis.toColorMap((0 until k).toArray) -) - -raster.tile.renderPng(clusterColors).write("target/scala-2.11/tut/ml/clustered.png") -``` - -| Color Composite | Cluster Assignments | -| ------------------ | ------------------- | -| ![](L8-RGB-VA.png) | ![](clustered.png) | - - -```tut:invisible -spark.stop() -``` diff --git a/docs/src/main/tut/ml/index.md b/docs/src/main/tut/ml/index.md deleted file mode 100644 index 5798ff3b7..000000000 --- a/docs/src/main/tut/ml/index.md +++ /dev/null @@ -1,18 +0,0 @@ -# Machine Learning - -@@@ note - -This examples here only scratch the surface of the type of machine learning one can perform -with RasterFrames. More examples are forthcoming. - -@@@ - -@@ toc { depth=2 } - -@@@ index - -* [Statistics](statistics.md) -* [Clustering](clustering.md) -* [Classification](classification.md) - -@@@ diff --git a/docs/src/main/tut/ml/statistics.md b/docs/src/main/tut/ml/statistics.md deleted file mode 100644 index 3ff086ad1..000000000 --- a/docs/src/main/tut/ml/statistics.md +++ /dev/null @@ -1,96 +0,0 @@ -# Raster Statistics - -```tut:invisible -import astraea.spark.rasterframes._ -import geotrellis.raster._ -import geotrellis.raster.render._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import geotrellis.spark._ -import geotrellis.spark.io._ -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ - -implicit val spark = SparkSession.builder(). - withKryoSerialization. - master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames -spark.sparkContext.setLogLevel("ERROR") -import spark.implicits._ -val scene = SinglebandGeoTiff("../core/src/test/resources/L8-B8-Robinson-IL.tiff") -val rf = scene.projectedRaster.toRF(128, 128).cache() -``` - -RasterFrames has a number of extension methods and columnar functions for performing analysis on tiles. - -## Tile Statistics - -### Tile Dimensions - -Get the nominal tile dimensions. Depending on the tiling there may be some tiles with different sizes on the edges. - -```tut -rf.select(rf.spatialKeyColumn, tile_dimensions($"tile")).show(3) -``` - -### Descriptive Statistics - -#### NoData Counts - -Count the numer of `NoData` and non-`NoData` cells in each tile. - -```tut -rf.select(rf.spatialKeyColumn, no_data_cells($"tile"), data_cells($"tile")).show(3) -``` - -#### Tile Mean - -Compute the mean value in each tile. Use `tileMean` for integral cell types, and `tileMeanDouble` for floating point -cell types. - -```tut -rf.select(rf.spatialKeyColumn, tile_mean($"tile")).show(3) -``` - -#### Tile Summary Statistics - -Compute a suite of summary statistics for each tile. Use `tile_stats` for integral cells types, and `tile_stats_double` -for floating point cell types. - -```tut -rf.withColumn("stats", tile_stats($"tile")).select(rf.spatialKeyColumn, $"stats.*").show(3) -``` - -### Histogram - -The `tile_histogram` function computes a histogram over the data in each tile. - -In this example we compute quantile breaks. - -```tut -rf.select(tile_histogram($"tile")).map(_.quantileBreaks(5)).show(5, false) -``` - -## Aggregate Statistics - -The `agg_stats` function computes the same summary statistics as `tile_stats`, but aggregates them over the whole -RasterFrame. - -```tut -rf.select(agg_stats($"tile")).show() -``` - -A more involved example: extract bin counts from a computed `Histogram`. - -```tut -rf.select(agg_approx_histogram($"tile")). - map(h => for(v <- h.labels) yield(v, h.itemCount(v))). - select(explode($"value") as "counts"). - select("counts._1", "counts._2"). - toDF("value", "count"). - orderBy(desc("count")). - show(10) -``` - -```tut:invisible -spark.stop() -``` - diff --git a/docs/src/main/tut/pyrasterframes.md b/docs/src/main/tut/pyrasterframes.md deleted file mode 100644 index 35a5169d5..000000000 --- a/docs/src/main/tut/pyrasterframes.md +++ /dev/null @@ -1,14 +0,0 @@ -# RasterFrames in Python - -RasterFrames includes Python bindings for using the library from within [PySpark](https://spark.apache.org/docs/latest/api/python/pyspark.html). -While creating Python documentation on parity with Scala is forthcoming, these resources should help -in the meantime: - -* [PyRasterFrames README](https://github.com/locationtech/rasterframes/blob/develop/pyrasterframes/python/README.rst) -* [PyRasterFrames Examples](https://github.com/locationtech/rasterframes/tree/develop/pyrasterframes/python/examples) -* [RasterFrames Jupyter Notebook](https://github.com/locationtech/rasterframes/blob/develop/deployment/README.md) -* @ref:[PyRasterFrames Functions](reference.md) - -Most features available in the Scala API are exposed in the Python API, refer to the @ref:[function reference](reference.md). Defining a [udf](http://spark.apache.org/docs/latest/api/python/pyspark.sql.html#pyspark.sql.functions.udf) using a `Tile` column through the Python API is not yet supported. - -If there's a specific feature that appears to be missing in the Python version [please submit an issue](https://github.com/locationtech/rasterframes/issues) so that we might address it for you. diff --git a/docs/src/main/tut/reference.md b/docs/src/main/tut/reference.md deleted file mode 100644 index d264d900d..000000000 --- a/docs/src/main/tut/reference.md +++ /dev/null @@ -1,946 +0,0 @@ -# Function Reference - -For the most up to date list of User Defined Functions using Tiles, look at API documentation for @scaladoc[`RasterFunctions`][RasterFunctions]. - -The full Scala API documentation can be found [here][scaladoc]. - -RasterFrames also provides SQL and Python bindings to many UDFs using the `Tile` column type. In Spark SQL, the functions are already registered in the SQL engine; they are usually prefixed with `rf_`. In Python, they are available in the `pyrasterframes.rasterfunctions` module. - -The convention in this document will be to define the function signature as below, with its return type, the function name, and named arguments with their types. - -``` -ReturnDataType function_name(InputDataType argument1, InputDataType argument2) -``` - -## List of Available SQL and Python Functions - -The convention in this document will be to define the function signature as below, with its return type, the function name, and named arguments with their types. - -``` -ReturnDataType function_name(InputDataType argument1, InputDataType argument2) -``` - -@@toc { depth=3 } - -### Vector Operations - -Various LocationTech GeoMesa UDFs to deal with `geomtery` type columns are also provided in the SQL engine and within the `pyrasterframes.rasterfunctions` Python module. These are documented in the [LocationTech GeoMesa Spark SQL documentation](https://www.geomesa.org/documentation/user/spark/sparksql_functions.html#). These functions are all prefixed with `st_`. - -RasterFrames provides two additional functions for vector geometry. - -#### reproject_geometry - -_Python_: - Geometry reproject_geometry(Geometry geom, String origin_crs, String destination_crs) - -_SQL_: `rf_reproject_geometry` - -Reproject the vector `geom` from `origin_crs` to `destination_crs`. Both `_crs` arguments are either [proj4](https://proj4.org/usage/quickstart.html) strings, [EPSG codes](https://www.epsg-registry.org/) codes or [OGC WKT](https://www.opengeospatial.org/standards/wkt-crs) for coordinate reference systems. - - -#### envelope - -_Python_: - - Struct[Double minX, Double maxX, Double minY, Double maxY] envelope(Geometry geom) - -Python only. Extracts the bounding box (envelope) of the geometry. - -See also GeoMesa [st_envelope](https://www.geomesa.org/documentation/user/spark/sparksql_functions.html#st-envelope) which returns a Geometry type. - -### Tile Metadata and Mutation - -Functions to access and change the particulars of a `tile`: its shape and the data type of its cells. See below section on @ref:[masking and nodata](reference.md#masking-and-nodata) for additional discussion of cell types. - -#### cell_types - - -_Python_: - - Array[String] cell_types() - -_SQL_: `rf_cell_types` - -Print an array of possible cell type names, as below. These names are used in other functions. See @ref:[discussion on nodata](reference.md#masking-and-nodata) for additional details. - -|cell_types | -|----------| -|bool | -|int8raw | -|int8 | -|uint8raw | -|uint8 | -|int16raw | -|int16 | -|uint16raw | -|uint16 | -|int32raw | -|int32 | -|float32raw| -|float32 | -|float64raw| -|float64 | - - -#### tile_dimensions - -_Python_: - - Struct[Int, Int] tile_dimensions(Tile tile) - -_SQL_: `rf_tile_dimensions` - -Get number of columns and rows in the `tile`, as a Struct of `cols` and `rows`. - -#### cell_type - -_Python_: - - Struct[String] cell_type(Tile tile) - -_SQL_: `rf_cell_type` - -Get the cell type of the `tile`. Available cell types can be retrieved with the @ref:[cell_types](reference.md#cell-types) function. - -#### convert_cell_type - -_Python_: - - Tile convert_cell_type(Tile tileCol, String cellType) - -_SQL_: `rf_convert_cell_type` - -Convert `tileCol` to a different cell type. - -#### resample - -_Python_: - - Tile resample(Tile tile, Double factor) - Tile resample(Tile tile, Int factor) - Tile resample(Tile tile, Tile shape_tile) - -_SQL_: `rf_resample` - -Change the tile dimension. Passing a numeric `factor` will scale the number of columns and rows in the tile: 1.0 is the same number of columns and row; less than one downsamples the tile; and greater than one upsamples the tile. Passing a `shape_tile` as the second argument outputs `tile` having the same number of columns and rows as `shape_tile`. All resampling is by nearest neighbor method. - -### Tile Creation - -Functions to create a new Tile column, either from scratch or from existing data not yet in a `tile`. - -#### tile_zeros - -_Python_: - -``` -Tile tile_zeros(Int tile_columns, Int tile_rows, String cell_type_name) -``` - -_SQL_: `rf_tile_zeros` - -Create a `tile` of shape `tile_columns` by `tile_rows` full of zeros, with the specified cell type. See function @ref:[`cell_types`](reference.md#cell-types) for valid values. All arguments are literal values and not column expressions. - -#### tile_ones - -_Python_: - -``` -Tile tile_ones(Int tile_columns, Int tile_rows, String cell_type_name) -``` - -_SQL_: `rf_tile_ones` - -Create a `tile` of shape `tile_columns` by `tile_rows` full of ones, with the specified cell type. See function @ref:[`cell_types`](reference.md#cell-types) for valid values. All arguments are literal values and not column expressions. - -#### make_constant_tile - -_Python_: - - Tile make_constant_tile(Numeric constant, Int tile_columns, Int tile_rows, String cell_type_name) - -_SQL_: `rf_make_constant_tile` - -Create a `tile` of shape `tile_columns` by `tile_rows` full of `constant`, with the specified cell type. See function @ref:[`cell_types`](reference.md#cell-types) for valid values. All arguments are literal values and not column expressions. - - -#### rasterize - -_Python_: - - Tile rasterize(Geometry geom, Geometry tile_bounds, Int value, Int tile_columns, Int tile_rows) - -_SQL_: `rf_rasterize` - -Convert a vector Geometry `geom` into a Tile representation. The `value` will be "burned-in" to the returned `tile` where the `geom` intersects the `tile_bounds`. Returned `tile` will have shape `tile_columns` by `tile_rows`. Values outside the `geom` will be assigned a nodata value. Returned `tile` has cell type `int32`, note that `value` is of type Int. - -Parameters `tile_columns` and `tile_rows` are literals, not column expressions. The others are column expressions. - - -Example use. In the code snip below, you can visualize the `tri` and `b` geometries with tools like [Wicket](https://arthur-e.github.io/Wicket/sandbox-gmaps3.html). The result is a right triangle burned into the `tile`, with nodata values shown as ∘. - - -```python -spark.sql(""" -SELECT rf_render_ascii( - rf_rasterize(tri, b, 8, 10, 10)) - -FROM - ( SELECT st_geomFromWKT('POLYGON((1.5 0.5, 1.5 1.5, 0.5 0.5, 1.5 0.5))') AS tri, - st_geomFromWKT('POLYGON((0.0 0.0, 2.0 0.0, 2.0 2.0, 0.0 2.0, 0.0 0.0))') AS b - ) r -""").show(1, False) - ------------ -|∘∘∘∘∘∘∘∘∘∘ -∘∘∘∘∘∘∘∘∘∘ -∘∘∘∘∘∘∘∘∘∘ -∘∘∘∘∘∘∘ ∘∘ -∘∘∘∘∘∘ ∘∘ -∘∘∘∘∘ ∘∘ -∘∘∘∘ ∘∘ -∘∘∘ ∘∘ -∘∘∘∘∘∘∘∘∘∘ -∘∘∘∘∘∘∘∘∘∘| ------------ -``` - - -#### array_to_tile - -_Python_: - - Tile array_to_tile(Array arrayCol, Int numCols, Int numRows) - -Python only. Create a `tile` from a Spark SQL [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), filling values in row-major order. - -#### assemble_tile - -_Python_: - - Tile assemble_tile(Int colIndex, Int rowIndex, Numeric cellData, Int numCols, Int numRows, String cellType) - -Python only. Create a Tile from a column of cell data with location indices. This function is the inverse of @ref:[`explode_tiles`](reference.md#explode-tiles). Intended use is with a `groupby`, producing one row with a new `tile` per group. The `numCols`, `numRows` and `cellType` arguments are literal values, others are column expressions. Valid values for `cellType` can be found with function @ref:[`cell_types`](reference.md#cell-types). - -### Masking and Nodata - -In raster operations, the preservation and correct processing of missing operations is very important. The idea of missing data is often expressed as a null or NaN. In raster data, missing observations are often termed NODATA; we will style them as nodata in this document. RasterFrames provides a variety of functions to manage and inspect nodata within `tile`s. - -See also statistical summaries to get the count of data and nodata values per `tile` and aggregate in a `tile` column: @ref:[`data_cells`](reference.md#data-cells), @ref:[`no_data_cells`](reference.md#no-data-cells), @ref:[`agg_data_cells`](reference.md#agg-data-cells), @ref:[`agg_no_data_cells`](reference.md#agg-no-data-cells). - -It is important to note that not all cell types support the nodata representation: these are `bool` and when the cell type string ends in `raw`. - -For integral valued cell types, the nodata is marked by a special sentinel value. This can be a default, typically zero or the minimum value for the underlying data type. The nodata value can also be a user-defined value. For example if the value 4 is to be interpreted as nodata, the cell type will read 'int32ud4'. - -For float cell types, the nodata can either be NaN or a user-defined value; for example `'float32ud-999.9'` would mean the value -999.9 is interpreted as a nodata. - -For more reading about cell types and ndodata, see the [GeoTrellis documentation](https://geotrellis.readthedocs.io/en/latest/guide/core-concepts.html?#working-with-cell-values). - -#### mask - -_Python_: - - Tile mask(Tile tile, Tile mask) - -_SQL_: `rf_mask` - -Where the `mask` contains nodata, replace values in the `tile` with nodata. - -Returned `tile` cell type will be coerced to one supporting nodata if it does not already. - - -#### inverse_mask - -_Python_: - - Tile inverse_mask(Tile tile, Tile mask) - -_SQL_: `rf_inverse_mask` - -Where the `mask` _does not_ contain nodata, replace values in `tile` with nodata. - -#### mask_by_value - -_Python_: - - Tile mask_by_value(Tile data_tile, Tile mask_tile, Int mask_value) - -_SQL_: `rf_mask_by_value` - -Generate a `tile` with the values from `data_tile`, with nodata in cells where the `mask_tile` is equal to `mask_value`. - - -#### is_no_data_tile - -_Python_: - - Boolean is_no_data_tile(tile) - -_SQL_: `rf_is_no_data_tile` - -Returns true if `tile` contains only nodata. By definition returns false if cell type does not support nodata. - -#### with_no_data - -_Python_: - - Tile with_no_data(Tile tile, Double no_data_value) - -Python only. Return a `tile` column marking as nodata all cells equal to `no_data_value`. - -The `no_data_value` argument is a literal Double, not a Column expression. - -If input `tile` had a nodata value already, the behaviour depends on if its cell type is floating point or not. For floating point cell type `tile`, nodata values on the input `tile` remain nodata values on the output. For integral cell type `tile`s, the previous nodata values become literal values. - -### Map Algebra - -[Map algebra](https://gisgeography.com/map-algebra-global-zonal-focal-local/) raster operations are element-wise operations between a `tile` and a scalar, between two `tile`s, or among many `tile`s. - -Some of these functions have similar variations in the Python API: - - - `local_op`: applies `op` to two columns; the right hand side can be a `tile` or a numeric column. - - `local_op_scalar`: applies `op` to a `tile` and a literal scalar, coercing the `tile` to a floating point type - - `local_op_scalar_int`: applies `op` to a `tile` and a literal scalar, without coercing the `tile` to a floating point type - -We will provide all these variations for `local_add` and then suppress the rest in this document. - -The SQL API does not require the `local_op_scalar` or `local_op_scalar_int` forms. - -#### local_add - -_Python_: - - Tile local_add(Tile tile1, Tile rhs) - Tile local_add(Tile tile1, Int rhs) - Tile local_add(Tile tile1, Double rhs) - -_SQL_: `rf_local_add` - -Returns a `tile` column containing the element-wise sum of `tile1` and `rhs`. - -#### local_add_scalar - -_Python_: - - Tile local_add_scalar(Tile tile, Double scalar) - -_SQL_: `rf_local_add_scalar` - -Returns a `tile` column containing the element-wise sum of `tile` and `scalar`. If `tile` is integral type, it will be coerced to floating before addition; returns float valued `tile`. - - -#### local_add_scalar_int - -_Python_: - - Tile local_add_scalar_int(Tile tile, Int scalar) - -_SQL_: `rf_local_add_scalar_int` - -Returns a `tile` column containing the element-wise sum of `tile` and `scalar`. If `tile` is integral type, returns integral type `tile`. - -#### local_subtract - -_Python_: - - Tile local_subtract(Tile tile1, Tile rhs) - Tile local_subtract(Tile tile1, Int rhs) - Tile local_subtract(Tile tile1, Double rhs) - -_SQL_: `rf_local_subtract` - -Returns a `tile` column containing the element-wise difference of `tile1` and `rhs`. - - -#### local_multiply - -_Python_: - - Tile local_multiply(Tile tile1, Tile rhs) - Tile local_multiply(Tile tile1, Int rhs) - Tile local_multiply(Tile tile1, Double rhs) - -_SQL_: `rf_local_multiply` - -Returns a `tile` column containing the element-wise product of `tile1` and `rhs`. This is **not** the matrix multiplication of `tile1` and `rhs`. - - -#### local_divide - -_Python_: - - Tile local_divide(Tile tile1, Tile rhs) - Tile local_divide(Tile tile1, Int rhs) - Tile local_divide(Tile tile1, Double rhs) - -_SQL_: `rf_local_divide` - -Returns a `tile` column containing the element-wise quotient of `tile1` and `rhs`. - - -#### normalized_difference - -_Python_: - - Tile normalized_difference(Tile tile1, Tile tile2) - -_SQL_: `rf_normalized_difference` - -Compute the normalized difference of the the two `tile`s: `(tile1 - tile2) / (tile1 + tile2)`. Result is always floating point cell type. This function has no scalar variant. - -#### local_less - -_Python_: - - Tile local_less(Tile tile1, Tile rhs) - Tile local_less(Tile tile1, Int rhs) - Tile local_less(Tile tile1, Double rhs) - -_SQL_: `rf_less` - -Returns a `tile` column containing the element-wise evaluation of `tile1` is less than `rhs`. - -#### local_less_equal - -_Python_: - - Tile local_less_equal(Tile tile1, Tile rhs) - Tile local_less_equal(Tile tile1, Int rhs) - Tile local_less_equal(Tile tile1, Double rhs) - -_SQL_: `rf_less_equal` - -Returns a `tile` column containing the element-wise evaluation of `tile1` is less than or equal to `rhs`. - -#### local_greater - -_Python_: - - Tile local_greater(Tile tile1, Tile rhs) - Tile local_greater(Tile tile1, Int rhs) - Tile local_greater(Tile tile1, Double rhs) - -_SQL_: `rf_greater` - -Returns a `tile` column containing the element-wise evaluation of `tile1` is greater than `rhs`. - -#### local_greater_equal - -_Python_: - - Tile local_greater_equal(Tile tile1, Tile rhs) - Tile local_greater_equal(Tile tile1, Int rhs) - Tile local_greater_equal(Tile tile1, Double rhs) - -_SQL_: `rf_greater_equal` - -Returns a `tile` column containing the element-wise evaluation of `tile1` is greater than or equal to `rhs`. - -#### local_equal - -_Python_: - - Tile local_equal(Tile tile1, Tile rhs) - Tile local_equal(Tile tile1, Int rhs) - Tile local_equal(Tile tile1, Double rhs) - -_SQL_: `rf_equal` - -Returns a `tile` column containing the element-wise equality of `tile1` and `rhs`. - -#### local_unequal - -_Python_: - - Tile local_unequal(Tile tile1, Tile rhs) - Tile local_unequal(Tile tile1, Int rhs) - Tile local_unequal(Tile tile1, Double rhs) - -_SQL_: `rf_unequal` - -Returns a `tile` column containing the element-wise inequality of `tile1` and `rhs`. - -#### round - -_Python_: - - Tile round(Tile tile) - -_SQL_: `rf_round` - -Round cell values to the nearest integer without changing the cell type. - -#### exp - -_Python_: - - Tile exp(Tile tile) - -_SQL_: `rf_exp` - -Performs cell-wise exponential. - -#### exp10 - -_Python_: - - Tile exp10(Tile tile) - -_SQL_: `rf_exp10` - -Compute 10 to the power of cell values. - -#### exp2 - -_Python_: - - Tile exp2(Tile tile) - -_SQL_: `rf_exp2` - -Compute 2 to the power of cell values. - -#### expm1 - -_Python_: - - Tile expm1(Tile tile) - -_SQL_: `rf_expm1` - -Performs cell-wise exponential, then subtract one. Inverse of @ref:[`log1p`](reference.md#log1p). - -#### log - -_Python_: - - Tile log(Tile tile) - -_SQL_: `rf_log` - -Performs cell-wise natural logarithm. - -#### log10 - -_Python_: - - Tile log10(Tile tile) - -_SQL_: `rf_log10` - -Performs cell-wise logarithm with base 10. - -#### log2 - -_Python_: - - Tile log2(Tile tile) - -_SQL_: `rf_log2` - -Performs cell-wise logarithm with base 2. - -#### log1p - -_Python_: - - Tile log1p(Tile tile) - -_SQL_: `rf_log1p` - -Performs natural logarithm of cell values plus one. Inverse of @ref:[`expm1`](reference.md#expm1). - -### Tile Statistics - -The following functions compute a statistical summary per row of a `tile` column. The statistics are computed across the cells of a single `tile`, within each DataFrame Row. Consider the following example. - -```python -import pyspark.functions as F -spark.sql(""" - SELECT 1 as id, rf_tile_ones(5, 5, 'float32') as t - UNION - SELECT 2 as id, rf_local_multiply(rf_tile_ones(5, 5, 'float32'), 3) as t - """).select(F.col('id'), tile_sum(F.col('t'))).show() - - -+---+-----------+ -| id|tile_sum(t)| -+---+-----------+ -| 2| 75.0| -| 1| 25.0| -+---+-----------+ -``` - - -#### tile_sum - -_Python_: - - Double tile_sum(Tile tile) - -_SQL_: `rf_tile_sum` - -Computes the sum of cells in each row of column `tile`, ignoring nodata values. - -#### tile_mean - -_Python_: - - Double tile_mean(Tile tile) - -_SQL_: `rf_tile_mean` - -Computes the mean of cells in each row of column `tile`, ignoring nodata values. - - -#### tile_min - -_Python_: - - Double tile_min(Tile tile) - -_SQL_: `rf_tile_min` - -Computes the min of cells in each row of column `tile`, ignoring nodata values. - - -#### tile_max - -_Python_: - - Double tile_max(Tile tile) - -_SQL_: `rf_tile_max` - -Computes the max of cells in each row of column `tile`, ignoring nodata values. - - -#### no_data_cells - -_Python_: - - Long no_data_cells(Tile tile) - -_SQL_: `rf_no_data_cells` - -Return the count of nodata cells in the `tile`. - -#### data_cells - -_Python_: - - Long data_cells(Tile tile) - -_SQL_: `rf_data_cells` - -Return the count of data cells in the `tile`. - -#### tile_stats - -_Python_: - - Struct[Long, Long, Double, Double, Double, Double] tile_stats(Tile tile) - -_SQL_: `tile_stats` - -Computes the following statistics of cells in each row of column `tile`: data cell count, nodata cell count, minimum, maximum, mean, and variance. The minimum, maximum, mean, and variance are computed ignoring nodata values. - - -#### tile_histogram - -_Python_: - - Struct[Struct[Long, Long, Double, Double, Double, Double], Array[Struct[Double, Long]]] tile_histogram(Tile tile) - -_SQL_: `rf_tile_histogram` - -Computes a statistical summary of cell values within each row of `tile`. Resulting column has the below schema. Note that several of the other `tile` statistics functions are convenience methods to extract parts of this result. Related is the @ref:[`agg_approx_histogram`](reference.md#agg-approx-histogram) which computes the statistics across all rows in a group. - -``` - |-- tile_histogram: struct (nullable = true) - | |-- stats: struct (nullable = true) - | | |-- dataCells: long (nullable = false) - | | |-- noDataCells: long (nullable = false) - | | |-- min: double (nullable = false) - | | |-- max: double (nullable = false) - | | |-- mean: double (nullable = false) - | | |-- variance: double (nullable = false) - | |-- bins: array (nullable = true) - | | |-- element: struct (containsNull = true) - | | | |-- value: double (nullable = false) - | | | |-- count: long (nullable = false) -``` - -### Aggregate Tile Statistics - -These functions compute statistical summaries over all of the cell values *and* across all the rows in the DataFrame or group. Example use below computes a single double-valued mean per month, across all data cells in the `red_band` `tile` type column. This would return at most twelve rows. - - -```python -from pyspark.functions import month -from pyrasterframes.functions import agg_mean -rf.groupby(month(rf.datetime)).agg(agg_mean(rf.red_band).alias('red_mean_monthly')) -``` - -Continuing our example from the @ref:[Tile Statistics](reference.md#tile-statistics) section, consider the following. Note that only a single row is returned. It is averaging 25 values of 1.0 and 25 values of 3.0, across the fifty cells in two rows. - -```python -spark.sql(""" -SELECT 1 as id, rf_tile_ones(5, 5, 'float32') as t -UNION -SELECT 2 as id, rf_local_multiply_scalar(rf_tile_ones(5, 5, 'float32'), 3) as t -""").agg(agg_mean(F.col('t'))).show(10, False) - -+-----------+ -|agg_mean(t)| -+-----------+ -|2.0 | -+-----------+ -``` - -#### agg_mean - -_Python_: - - Double agg_mean(Tile tile) - -_SQL_: @ref:[`rf_agg_stats`](reference.md#agg-stats)`(tile).mean` - -Aggregates over the `tile` and return the mean of cell values, ignoring nodata. Equivalent to @ref:[`agg_stats`](reference.md#agg-stats)`.mean`. - - -#### agg_data_cells - -_Python_: - - Long agg_data_cells(Tile tile) - -_SQL_: @ref:[`rf_agg_stats`](reference.md#agg-stats)`(tile).dataCells` - -Aggregates over the `tile` and return the count of data cells. Equivalent to @ref:[`agg_stats`](reference.md#agg-stats)`.dataCells`. C.F. `data_cells`; equivalent code: - -```python -rf.select(agg_data_cells(rf.tile).alias('agg_data_cell')).show() -# Equivalent to -rf.agg(F.sum(data_cells(rf.tile)).alias('agg_data_cell')).show() -``` - -#### agg_no_data_cells - -_Python_: - - Long agg_no_data_cells(Tile tile) - -_SQL_: @ref:[`rf_agg_stats`](reference.md#agg-stats)`(tile).noDataCells` - -Aggregates over the `tile` and return the count of nodata cells. Equivalent to @ref:[`agg_stats`](reference.md#agg-stats)`.noDataCells`. C.F. @ref:[`no_data_cells`](reference.md#no-data-cells) a row-wise count of no data cells. - -#### agg_stats - -_Python_: - - Struct[Long, Long, Double, Double, Double, Double] agg_stats(Tile tile) - -_SQL_: `rf_agg_stats` - -Aggregates over the `tile` and returns statistical summaries of cell values: number of data cells, number of nodata cells, minimum, maximum, mean, and variance. The minimum, maximum, mean, and variance ignore the presence of nodata. - -#### agg_approx_histogram - -_Python_: - - Struct[Struct[Long, Long, Double, Double, Double, Double], Array[Struct[Double, Long]]] agg_approx_histogram(Tile tile) - -_SQL_: `rf_agg_approx_histogram` - -Aggregates over the `tile` return statistical summaries of the cell values, including a histogram, in the below schema. The `bins` array is of tuples of histogram values and counts. Typically values are plotted on the x-axis and counts on the y-axis. - -Note that several of the other cell value statistics functions are convenience methods to extract parts of this result. Related is the @ref:[`tile_histogram`](reference.md#tile-histogram) function which operates on a single row at a time. - -``` - |-- agg_approx_histogram: struct (nullable = true) - | |-- stats: struct (nullable = true) - | | |-- dataCells: long (nullable = false) - | | |-- noDataCells: long (nullable = false) - | | |-- min: double (nullable = false) - | | |-- max: double (nullable = false) - | | |-- mean: double (nullable = false) - | | |-- variance: double (nullable = false) - | |-- bins: array (nullable = true) - | | |-- element: struct (containsNull = true) - | | | |-- value: double (nullable = false) - | | | |-- count: long (nullable = false) -``` - -### Tile Local Aggregate Statistics - -Local statistics compute the element-wise statistics across a DataFrame or group of `tile`s, resulting in a `tile` that has the same dimension. - -Consider again our example for Tile Statistics and Aggregate Tile Statistics, this time apply @ref:[`agg_local_mean`](reference.md#agg-local-mean). We see that it is computing the element-wise mean across the two rows. In this case it is computing the mean of one value of 1.0 and one value of 3.0 to arrive at the element-wise mean, but doing so twenty-five times, one for each position in the `tile`. - - -```python -import pyspark.functions as F -lam = spark.sql(""" -SELECT 1 as id, rf_tile_ones(5, 5, 'float32') as t -UNION -SELECT 2 as id, rf_local_multiply(rf_tile_ones(5, 5, 'float32'), 3) as t -""").agg(local_agg_mean(F.col('t')).alias('l')) \ - -## local_agg_mean returns a tile -lam.select(tile_dimensions(lam.l)).show() -## -+------------------+ -|tile_dimensions(l)| -+------------------+ -| [5, 5]| -+------------------+ -## - -lam.select(explode_tiles(lam.l)).show(10, False) -## -+------------+---------+---+ -|column_index|row_index|l | -+------------+---------+---+ -|0 |0 |2.0| -|1 |0 |2.0| -|2 |0 |2.0| -|3 |0 |2.0| -|4 |0 |2.0| -|0 |1 |2.0| -|1 |1 |2.0| -|2 |1 |2.0| -|3 |1 |2.0| -|4 |1 |2.0| -+------------+---------+---+ -only showing top 10 rows -``` - - -#### agg_local_max - -_Python_: - - Tile agg_local_max(Tile tile) - -_SQL_: `rf_agg_local_max` - -Compute the cell-local maximum operation over Tiles in a column. - -#### agg_local_min - -_Python_: - - Tile agg_local_min(Tile tile) - -_SQL_: `rf_agg_local_min` - -Compute the cell-local minimum operation over Tiles in a column. - -#### agg_local_mean - -_Python_: - - Tile agg_local_mean(Tile tile) - -_SQL_: `rf_agg_local_mean` - -Compute the cell-local mean operation over Tiles in a column. - -#### agg_local_data_cells - -_Python_: - - Tile agg_local_data_cells(Tile tile) - -_SQL_: `rf_agg_local_data_cells` - -Compute the cell-local count of data cells over Tiles in a column. Returned `tile` has a cell type of `int32`. - -#### agg_local_no_data_cells - -_Python_: - - Tile agg_local_no_data_cells(Tile tile) - -_SQL_: `rf_agg_local_no_data_cells` - -Compute the cell-local count of nodata cells over Tiles in a column. Returned `tile` has a cell type of `int32`. - -#### agg_local_stats - -_Python_: - - Struct[Tile, Tile, Tile, Tile, Tile] agg_local_stats(Tile tile) - -_SQL_: `rf_agg_local_stats` - -Compute cell-local aggregate count, minimum, maximum, mean, and variance for a column of Tiles. Returns a struct of five `tile`s. - - -### Converting Tiles - -RasterFrames provides several ways to convert a `tile` into other data structures. See also functions for @ref:[creating tiles](reference.md#tile-creation). - -#### explode_tiles - -_Python_: - - Int, Int, Numeric* explode_tiles(Tile* tile) - -_SQL_: `rf_explode_tiles` - -Create a row for each cell in `tile` columns. Many `tile` columns can be passed in, and the returned DataFrame will have one numeric column per input. There will also be columns for `column_index` and `row_index`. Inverse of @ref:[`assemble_tile`](reference.md#assemble-tile). When using this function, be sure to have a unique identifier for rows in order to successfully invert the operation. - -#### explode_tiles_sample - -_Python_: - - Int, Int, Numeric* explode_tiles_sample(Double sample_frac, Long seed, Tile* tile) - -Python only. As with @ref:[`explode_tiles`](reference.md#explode-tiles), but taking a randomly sampled subset of cells. Equivalent to the below, but this implementation is optimized for speed. Parameter `sample_frac` should be between 0.0 and 1.0. - -```python -df.select(df.id, explode_tiles(df.tile1, df.tile2, df.tile3)) \ - .sample(False, 0.05, 8675309) -# Equivalent result, faster -df.select(df.id, explode_tiles_sample(0.05, 8675309, df.tile1, df.tile2, df.tile3)) \ -``` - -#### tile_to_int_array - -_Python_: - - Array tile_to_int_array(Tile tile) - -_SQL_: `rf_tile_to_int_array` - - -Convert Tile column to Spark SQL [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), in row-major order. Float cell types will be coerced to integral type by flooring. - - -#### tile_to_double_array - -_Python_: - - Array tile_to_double_arry(Tile tile) - -_SQL_: `rf_tile_to_double_array` - -Convert tile column to Spark [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), in row-major order. Integral cell types will be coerced to floats. - - -#### render_ascii - -_Python_: - - String render_ascii(Tile tile) - -_SQL_: `rf_render_ascii` - -Pretty print the tile values as plain text. - - - -[RasterFunctions]: astraea.spark.rasterframes.RasterFunctions -[scaladoc]: latest/api/index.html - diff --git a/docs/src/main/tut/spatial-queries.md b/docs/src/main/tut/spatial-queries.md deleted file mode 100644 index c492b114a..000000000 --- a/docs/src/main/tut/spatial-queries.md +++ /dev/null @@ -1,7 +0,0 @@ -# Spatial Queries - -An important aspect of working with big geospatial data is the ability to filter based on spatio-temporal predicates. RasterFrames uses [GeoMesa Spark JTS](http://www.geomesa.org/documentation/current/user/spark/spark_jts.html) for specifying and executing spatial queries against RasterFrame data sources. - -@@@ note -Until this section is written, please see [the `GeoTrellisDataSource` tests](https://github.com/s22s/raster-frames/blob/develop/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala) for examples on how to use the spatial query support. -@@@ diff --git a/experimental/build.sbt b/experimental/build.sbt deleted file mode 100644 index 2b5168fbc..000000000 --- a/experimental/build.sbt +++ /dev/null @@ -1,12 +0,0 @@ -moduleName := "rasterframes-experimental" - -libraryDependencies ++= Seq( - geotrellis("s3").value, - spark("core").value % Provided, - spark("mllib").value % Provided, - spark("sql").value % Provided -) - -fork in IntegrationTest := true -javaOptions in IntegrationTest := Seq("-Xmx2G") -parallelExecution in IntegrationTest := false diff --git a/experimental/src/it/resources/larger-example.geojson b/experimental/src/it/resources/larger-example.geojson deleted file mode 100644 index d0a3b1b82..000000000 --- a/experimental/src/it/resources/larger-example.geojson +++ /dev/null @@ -1,91 +0,0 @@ -{ - "type": "FeatureCollection", - "crs": { - "type": "name", - "properties": { - "name": "EPSG:3857" - } - }, - "features": [{ - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [0, 0] - }, - "properties": { - "ID": 1 - } - }, { - "type": "Feature", - "geometry": { - "type": "LineString", - "coordinates": [[4e6, -2e6], [8e6, 2e6]] - }, - "properties": { - "ID": 2 - } - }, { - "type": "Feature", - "geometry": { - "type": "LineString", - "coordinates": [[4e6, 2e6], [8e6, -2e6]] - }, - "properties": { - "ID": 3 - } - }, { - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [[[-5e6, -1e6], [-4e6, 1e6], [-3e6, -1e6],[-5e6, -1e6]]] - }, - "properties": { - "ID": 4 - } - }, { - "type": "Feature", - "geometry": { - "type": "MultiLineString", - "coordinates": [ - [[-1e6, -7.5e5], [-1e6, 7.5e5]], - [[1e6, -7.5e5], [1e6, 7.5e5]], - [[-7.5e5, -1e6], [7.5e5, -1e6]], - [[-7.5e5, 1e6], [7.5e5, 1e6]] - ] - }, - "properties": { - "ID": 5 - } - }, { - "type": "Feature", - "geometry": { - "type": "MultiPolygon", - "coordinates": [ - [[[-5e6, 6e6], [-5e6, 8e6], [-3e6, 8e6], [-3e6, 6e6]]], - [[[-2e6, 6e6], [-2e6, 8e6], [0, 8e6], [0, 6e6]]], - [[[1e6, 6e6], [1e6, 8e6], [3e6, 8e6], [3e6, 6e6]]] - ] - }, - "properties": { - "ID": 6 - } - }, { - "type": "Feature", - "geometry": { - "type": "GeometryCollection", - "geometries": [{ - "type": "LineString", - "coordinates": [[-5e6, -5e6], [0, -5e6]] - }, { - "type": "Point", - "coordinates": [4e6, -5e6] - }, { - "type": "Polygon", - "coordinates": [[[1e6, -6e6], [2e6, -4e6], [3e6, -6e6]]] - }] - }, - "properties": { - "ID": 7 - } - }] -} \ No newline at end of file diff --git a/experimental/src/it/resources/log4j.properties b/experimental/src/it/resources/log4j.properties index 6a7d4154e..5cc16f4db 100644 --- a/experimental/src/it/resources/log4j.properties +++ b/experimental/src/it/resources/log4j.properties @@ -35,7 +35,7 @@ log4j.logger.org.spark_project.jetty=WARN log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO -log4j.logger.astraea.spark.rasterframes=DEBUG +log4j.logger.org.locationtech.rasterframes=DEBUG log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support diff --git a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala b/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala deleted file mode 100644 index e86376246..000000000 --- a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource.awspds - -import java.net.URL - -import astraea.spark.rasterframes.TestEnvironment -import org.apache.spark.sql.functions._ -import astraea.spark.rasterframes.experimental.datasource._ - -/** - * Test rig for L8 catalog stuff. - * - * @since 5/4/18 - */ -class L8CatalogRelationTest extends TestEnvironment { - describe("Representing L8 scenes as a Spark data source") { - import spark.implicits._ - val catalog = spark.read.format(L8CatalogDataSource.SHORT_NAME).load() - - val scenes = catalog - .where($"acquisition_date" === to_timestamp(lit("2017-04-04 15:12:55.394"))) - .where($"path" === 11 && $"row" === 12) - - it("should provide a non-empty catalog") { - assert(scenes.count() === 1) - } - - it("should construct band specific download URLs") { - val b01 = scenes.select(l8_band_url("B1")) - noException shouldBe thrownBy { - new URL(b01.first()) - } - } - - it("should download geotiff as blob") { - import org.apache.spark.sql.functions.{length ⇒ alength} - val b01 = scenes.limit(1) - .select(download(l8_band_url("B1")) as "data") - - val len = b01.select(alength($"data").as[Long]) - assert(len.first() >= 4000000) - } - - it("should download geotiff as tiles") { - val b01 = scenes - .select($"*", read_tiles(l8_band_url("B1"))) - assert(b01.count() === 1089) - } - } -} diff --git a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8RelationTest.scala b/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8RelationTest.scala deleted file mode 100644 index 688866a93..000000000 --- a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8RelationTest.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource.awspds - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.util.ReadAccumulator -import org.apache.spark.sql.DataFrame -import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} - -/** - * @since 8/21/18 - */ -class L8RelationTest extends TestEnvironment with BeforeAndAfterAll with BeforeAndAfter { - - private var scenes: DataFrame = _ - - val query = """ - |SELECT bounds, timestamp, B1, B2 - |FROM l8 - |WHERE - | st_intersects(bounds, st_geomFromText('LINESTRING (-39.551 -7.1881, -72.2461 -45.7062)')) AND - | timestamp > to_timestamp('2017-11-01') AND - | timestamp <= to_timestamp('2017-11-03') - """.stripMargin - - override protected def beforeAll(): Unit = { - val l8 = spark.read - .format(L8DataSource.SHORT_NAME) - .option(L8DataSource.ACCUMULATORS, true) - .load() - l8.createOrReplaceTempView("l8") - scenes = sql(query).cache() - } - - describe("Read L8 on PDS as a DataSource") { - it("should count scenes") { - assert(scenes.schema.size === 4) - assert(scenes.count() === 7) - } - - it("should count tiles") { - val l8 = spark.read - .format(L8DataSource.SHORT_NAME) - .option(L8DataSource.ACCUMULATORS, true) - .option(L8DataSource.USE_TILING, true) - .load() - l8.createOrReplaceTempView("l82") - val scenes2 = sql(query.replaceAll("l8", "l82")).cache() - val scenesCount = scenes.count() - val scenes2Count = scenes2.count() - println(scenesCount, scenes2Count) - // Most L8 geotiffs are 16x16 tiles, but some are smaller. - // Test against the lower bound. - assert(scenes2Count > scenesCount * 15 * 15) - } - } -} diff --git a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala b/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala deleted file mode 100644 index eb2771e6f..000000000 --- a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource.awspds -import java.net.URL -import java.sql.Timestamp - -import astraea.spark.rasterframes.TestEnvironment -import astraea.spark.rasterframes.experimental.datasource._ -import org.apache.spark.sql.functions._ - -/** - * Test rig for MODIS catalog stuff. - * - * @since 5/4/18 - */ -class MODISCatalogRelationTest extends TestEnvironment { - describe("Representing MODIS scenes as a Spark data source") { - import spark.implicits._ - val catalog = spark.read.format(MODISCatalogDataSource.SHORT_NAME).load() - val scenes = catalog - .where($"acquisition_date".as[Timestamp] === to_timestamp(lit("2018-1-1"))) - .where($"granule_id".contains("h24v03")) - .cache() - - it("should provide a non-empty catalog") { - scenes.show(false) - assert(scenes.count() === 1) - } - - it("should construct band specific download URLs") { - val b01 = scenes.select($"assets"("B01").as[String]) - b01.show(false) - noException shouldBe thrownBy { - new URL(b01.first()) - } - } - - it("should download geotiff as blob") { - import org.apache.spark.sql.functions.{length ⇒ alength} - val b01 = scenes.limit(1) - .select(download($"assets"("B01")) as "data") - - val len = b01.select(alength($"data").as[Long]) - assert(len.first() >= 4000000) - } - - it("should download geotiff as tiles") { - val b01 = scenes - .select(read_tiles($"assets"("B01") as "B01", $"assets"("B02") as "B02")) - b01.show(false) - assert(b01.count() === 100) - -// val kv = b01.select($"B01_extent", $"B01_tile").as[(Extent, Tile)] -// kv.collect.zipWithIndex.foreach { case ((extent, tile), idx) ⇒ -// GeoTiff(tile, extent, Sinusoidal).write(s"target/b01-tile-$idx.tiff") -// } - } - } -} diff --git a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala b/experimental/src/it/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala deleted file mode 100644 index 64e3c43f4..000000000 --- a/experimental/src/it/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.slippy - -import java.io.File - -import astraea.spark.rasterframes._ -import geotrellis.raster._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.sql.SparkSession -import SlippyExport._ -import astraea.spark.rasterframes.util.MultibandRender - -object SlippyExportDriver { - def main(args: Array[String]): Unit = { - - implicit val spark = SparkSession - .builder() - .master("local[*]") - .appName("RasterFrames") - .getOrCreate() - .withRasterFrames - - def mergeBands = { - val bands: Seq[SinglebandGeoTiff] = for (i ← 1 to 3) yield { - TestData.l8Sample(i) - } - - val mtile = MultibandTile(bands.map(_.tile)) - - val pr = ProjectedRaster(mtile, bands.head.extent, bands.head.crs) - - implicit val bandCount = PairRDDConverter.forSpatialMultiband(bands.length) - - val rf = pr.toRF(64, 64) - - //rf.exportGeoTiffTiles(new File("target/slippy-tiff").toURI) - - rf.exportSlippyMap(new File("target/slippy-1/").toURI) - } - - def multiband = { - implicit val bandCount = PairRDDConverter.forSpatialMultiband(3) - val rf = TestData.rgbCogSample.projectedRaster.toRF(256, 256) - import astraea.spark.rasterframes.datasource.geotiff._ - //val rf = spark.read.geotiff.loadRF(getClass.getResource("/LC08_RGB_Norfolk_COG.tiff").toURI) - println(rf.tileLayerMetadata.merge.toString) - rf.exportSlippyMap(new File("target/slippy-2/").toURI, MultibandRender.Landsat8NaturalColor) - } - - multiband - //mergeBands - } -} diff --git a/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala new file mode 100644 index 000000000..9b6d8480f --- /dev/null +++ b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelationTest.scala @@ -0,0 +1,108 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea. Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * + */ + +package org.locationtech.rasterframes.experimental.datasource.awspds + +import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ + +/** + * Test rig for L8 catalog stuff. + * + * @since 5/4/18 + */ +class L8CatalogRelationTest extends TestEnvironment { + import spark.implicits._ + + val catalog = spark.read.l8Catalog.load() + + val scenes = catalog + .where($"acquisition_date" === to_timestamp(lit("2017-04-04 15:12:55.394"))) + .where($"path" === 11 && $"row" === 12) + .cache() + + describe("Representing L8 scenes as a Spark data source") { + it("should provide a non-empty catalog") { + scenes.count() shouldBe 1 + } + + it("should provide 11 band + 1 QA urls") { + scenes.schema.count(_.name.startsWith("B")) shouldBe 12 + } + + it("should construct valid URLs") { + val urlStr = scenes.select("B11").as[String].first + val code = TestSupport.urlResponse(urlStr) + code should be (200) + } + + it("should work with SQL and spatial predicates") { + catalog.createOrReplaceTempView("l8_catalog") + val scenes = spark.sql(""" + SELECT st_geometry(bounds_wgs84) as geometry, acquisition_date, B1, B2 + FROM l8_catalog + WHERE + st_intersects(st_geometry(bounds_wgs84), st_geomFromText('LINESTRING (-39.551 -7.1881, -72.2461 -45.7062)')) AND + acquisition_date > to_timestamp('2017-11-01') AND + acquisition_date <= to_timestamp('2017-12-13') + """) + + scenes.count() shouldBe > (100L) + } + + it("should construct expected extents") { + catalog.createOrReplaceTempView("l8_catalog") + + catalog.filter($"bounds_wgs84.xmin" > $"bounds_wgs84.xmax").count() shouldBe (0) + catalog.filter($"bounds_wgs84.ymin" > $"bounds_wgs84.ymax").count() shouldBe (0) + + val geo_area_row = spark.sql( + """ + SELECT min(st_area(st_geometry(bounds_wgs84))) AS area + FROM l8_catalog + WHERE st_intersects(st_geometry(bounds_wgs84), st_geomFromText('LINESTRING(-78.035 39.004,-80.166 37.241)')) AND + acquisition_date > to_timestamp('2017-11-01') AND + acquisition_date <= to_timestamp('2017-11-16') + """).first() + val geo_area = geo_area_row.getDouble(0) + geo_area shouldBe < (6.5) + geo_area shouldBe > (4.5) + } + } + + describe("Read L8 scenes from PDS") { + it("should be compatible with raster DataSource") { + val df = spark.read.raster + .fromCatalog(scenes, "B1", "B3") + .withTileDimensions(512, 512) + .load() + + // Further refine down to a tile + val sub = df.select($"B3") + .where(st_contains(st_geometry(rf_extent($"B1")), st_makePoint(574965, 7679175))) + + val stats = sub.select(rf_agg_stats($"B3")).first + + stats.data_cells should be (512L * 512L) + stats.mean shouldBe > (10000.0) + } + } +} diff --git a/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala new file mode 100644 index 000000000..3f777d322 --- /dev/null +++ b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelationTest.scala @@ -0,0 +1,79 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea. Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * + */ + +package org.locationtech.rasterframes.experimental.datasource.awspds +import java.sql.Timestamp + +import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ +import org.locationtech.rasterframes.stats.CellStatistics + +/** + * Test rig for MODIS catalog stuff. + * + * @since 5/4/18 + */ +class MODISCatalogRelationTest extends TestEnvironment { + import spark.implicits._ + val catalog = spark.read.modisCatalog.load() + val scenes = catalog + .where($"acquisition_date".as[Timestamp] === to_timestamp(lit("2018-1-1"))) + .where($"granule_id".contains("h24v03")) + .cache() + + describe("Representing MODIS scenes as a Spark data source") { + it("should provide a non-empty catalog") { + scenes.count() should be (1) + } + + it("should provide 7 band and 7 qa urls") { + scenes.schema.count(_.name.startsWith("B")) should be (14) + } + + it("should construct valid URLs") { + val urlStr = scenes.select("B03").as[String].first + val code = TestSupport.urlResponse(urlStr) + withClue(urlStr) { + code should be(200) + } + } + } + + describe("Read MODIS scenes from PDS") { + it("should be compatible with raster DataSource") { + val df = spark.read.raster + .fromCatalog(scenes, "B03") + .withTileDimensions(128, 128) + .load() + + // Further refine down to a tile + val sub = df.select($"B03", st_centroid(st_geometry(rf_extent($"B03")))) + .where(st_contains(st_geometry(rf_extent($"B03")), st_makePoint(7175787.353582373, 6345530.965564346))) + .withColumn("stats", rf_tile_stats(rf_tile($"B03"))) + + val stats = sub.select($"stats".as[CellStatistics]).first() + + stats.data_cells shouldBe < (128L * 128L) + stats.data_cells shouldBe > (128L) + stats.mean shouldBe > (1000.0) + } + } +} diff --git a/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/TestSupport.scala b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/TestSupport.scala new file mode 100644 index 000000000..a8d33f77e --- /dev/null +++ b/experimental/src/it/scala/org/locationtech/rasterframes/experimental/datasource/awspds/TestSupport.scala @@ -0,0 +1,38 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.experimental.datasource.awspds + +import java.net.{HttpURLConnection, URL} + +object TestSupport { + def urlResponse(urlStr: String): Int = { + val conn = new URL(urlStr).openConnection().asInstanceOf[HttpURLConnection] + try { + conn.setRequestMethod("GET") + conn.connect() + conn.getResponseCode + } + finally { + conn.disconnect() + } + } +} diff --git a/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index 99180fa88..97275f043 100644 --- a/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1,4 +1,2 @@ -astraea.spark.rasterframes.experimental.datasource.geojson.GeoJsonDataSource -astraea.spark.rasterframes.experimental.datasource.awspds.L8CatalogDataSource -astraea.spark.rasterframes.experimental.datasource.awspds.MODISCatalogDataSource -astraea.spark.rasterframes.experimental.datasource.awspds.L8DataSource +org.locationtech.rasterframes.experimental.datasource.awspds.L8CatalogDataSource +org.locationtech.rasterframes.experimental.datasource.awspds.MODISCatalogDataSource diff --git a/experimental/src/main/resources/slippy.html b/experimental/src/main/resources/slippy.html deleted file mode 100644 index 4f616928f..000000000 --- a/experimental/src/main/resources/slippy.html +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - RasterFrames - - - - - - - - - - -
                          - - - - diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadExpression.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadExpression.scala deleted file mode 100644 index 32d55645b..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadExpression.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource - -import java.net.URI - -import astraea.spark.rasterframes.util.GeoTiffInfoSupport -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.sql.{Column, TypedColumn} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{Expression, Generator, GenericInternalRow, UnaryExpression} -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String -import astraea.spark.rasterframes.util._ - -/** - * Downloads data from URL and stores it in a column. - * - * @since 5/4/18 - */ -case class DownloadExpression(override val child: Expression, colPrefix: String) extends UnaryExpression - with Generator with CodegenFallback with GeoTiffInfoSupport with DownloadSupport with LazyLogging { - - override def nodeName: String = "download" - - override def checkInputDataTypes(): TypeCheckResult = { - if(child.dataType == StringType) TypeCheckSuccess - else TypeCheckFailure( - s"Expected '${StringType.typeName}' but received '${child.dataType.simpleString}'" - ) - } - - override def elementSchema: StructType = StructType(Seq( - StructField(colPrefix + "_data", BinaryType, true) - )) - - override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - val urlString = child.eval(input).asInstanceOf[UTF8String] - val bytes = getBytes(URI.create(urlString.toString)) - Traversable(new GenericInternalRow(Array[Any](bytes))) - } -} - -object DownloadExpression { - import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc - - def apply(urlColumn: Column): TypedColumn[Any, Array[Byte]] = - new Column( - new DownloadExpression(urlColumn.expr, urlColumn.columnName) - ).as[Array[Byte]] -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/ReadTilesExpression.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/ReadTilesExpression.scala deleted file mode 100644 index 1d00be0d5..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/ReadTilesExpression.scala +++ /dev/null @@ -1,167 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource - -import java.net.URI - -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.ref.HttpRangeReader -import com.typesafe.scalalogging.LazyLogging -import geotrellis.proj4.CRS -import geotrellis.raster.{ProjectedRaster, Tile} -import geotrellis.raster.io.geotiff.reader.GeoTiffReader -import geotrellis.spark.io.hadoop.HdfsRangeReader -import geotrellis.spark.io.s3.S3Client -import geotrellis.spark.io.s3.util.S3RangeReader -import geotrellis.util.{ByteReader, FileRangeReader, RangeReader, StreamingByteReader} -import geotrellis.vector.Extent -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, Generator, Literal} -import org.apache.spark.sql.rf._ -import org.apache.spark.sql.types._ - -/** - * Catalyst generator to convert a geotiff download URL into a series of rows containing the internal - * tiles and associated extents. - * - * @since 5/4/18 - */ -case class ReadTilesExpression(children: Seq[Expression]) extends Expression - with Generator with CodegenFallback with DownloadSupport with LazyLogging { - - private val TileType = new TileUDT() - - override def nodeName: String = "download_tiles" - - def inputTypes = Seq.fill(children.length)(StringType) - - override def elementSchema: StructType = StructType(Seq( - StructField("crs", CatalystSerializer[CRS].schema, true), - StructField("extent", CatalystSerializer[Extent].schema, true) - ) ++ - children - .zipWithIndex - .map { - case (l: Literal, _) ⇒ String.valueOf(l.value) - case (a: Alias, _) ⇒ a.name - case (_, i) ⇒ s"_${i + 1}" - } - .map(name ⇒ { - StructField(name, TileType, true) - }) - ) - - private def reader(uri: URI): ByteReader = { - val rr: RangeReader = uri.getScheme match { - case "http" | "https" ⇒ HttpRangeReader(uri) - case "file" ⇒ FileRangeReader(uri.getPath) - case "hdfs" | "s3n" | "s3a" | "wasb" | "wasbs" ⇒ - HdfsRangeReader(new Path(uri), new Configuration()) - case "s3" ⇒ - S3RangeReader(uri, S3Client.DEFAULT) - } - StreamingByteReader(rr) - } - - def checkDimensions(tiles: Array[Array[ProjectedRaster[Tile]]], uris: Seq[URI]) = { - val rowDims = tiles.map(row ⇒ { - row.map { - case null ⇒ null - case t ⇒ t.tile.dimensions - } - }) - - val notSameDim = rowDims - .map(_.distinct.count(_ != null)) - .indexWhere(_ > 1) - - if (notSameDim >= 0) { - throw new IllegalArgumentException("Detecuris: Seq[Option[String]] ted rows with different sized tiles: " + - rowDims(notSameDim).mkString(", ") + "\nfrom: " + uris.mkString(", ")) - } - } - - - private def safeTranspose(tiles: Seq[Seq[ProjectedRaster[Tile]]]) = { - // We need to transpose the nested sequence of from column major to row major order - // However, some columns have no rows, the default transpose needs a regular grid... - // so we do it manually. - val numRows = tiles.map(_.size).max - val numCols = tiles.length - val tileGrid = Array.fill(numRows)(Array.ofDim[ProjectedRaster[Tile]](numCols)) - - for { - (rows, col) ← tiles.zipWithIndex - (tile, row) ← rows.zipWithIndex - } { - tileGrid(row)(col) = tile - } - tileGrid - } - - override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - val uris = children.map(_.eval(input).toString).map(URI.create) - val colMajor = uris.map { uri ⇒ - val tiff = GeoTiffReader.readSingleband(reader(uri)) - val segmentLayout = tiff.imageData.segmentLayout - val windows = segmentLayout.listWindows(256) - val re = tiff.rasterExtent - - val subtiles = tiff.crop(windows) - for { - (gridbounds, tile) ← subtiles.toSeq - } yield { - val extent = re.extentFor(gridbounds, false) - ProjectedRaster(tile, extent, tiff.crs) - } - } - - val rowMajor = safeTranspose(colMajor) - - checkDimensions(rowMajor, uris) - - rowMajor.map(row ⇒ { - val serializedTiles = row.map { - case null ⇒ null - case t ⇒ TileType.serialize(t) - } - - val firstBandExt = row - .find(_ != null) - .map(p ⇒ Seq(p.crs.toInternalRow, p.extent.toInternalRow)) - .getOrElse(Seq(null, null)) - - InternalRow(firstBandExt ++ serializedTiles: _*) - }) - } - -} - -object ReadTilesExpression { - def apply(urls: Seq[Column]): Column = new Column( - new ReadTilesExpression(urls.map(_.expr)) - ) -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8DataSource.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8DataSource.scala deleted file mode 100644 index 9933ec256..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8DataSource.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource.awspds - -import astraea.spark.rasterframes.util.ReadAccumulator -import org.apache.spark.sql.SQLContext -import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationProvider} - -/** - * Experiment in providing RasterFrames over whole PDS catalog. - * - * @since 8/21/18 - */ -class L8DataSource extends DataSourceRegister with RelationProvider { - override def shortName(): String = L8DataSource.SHORT_NAME - - override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { - val useTiling = parameters.get(L8DataSource.USE_TILING).exists(_.toBoolean) - val accumulators = parameters - .get(L8DataSource.ACCUMULATORS) - .filter(_.toBoolean) - .map(_ ⇒ ReadAccumulator(sqlContext.sparkContext, L8DataSource.SHORT_NAME)) - L8Relation(sqlContext, useTiling, accumulators) - } -} - -object L8DataSource { - final val SHORT_NAME = "awsl8" - final val USE_TILING = "useTiling" - final val ACCUMULATORS = "accumulators" -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8Relation.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8Relation.scala deleted file mode 100644 index 845f5aad3..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8Relation.scala +++ /dev/null @@ -1,134 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource.awspds - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.encoders.CatalystSerializer -import astraea.spark.rasterframes.encoders.CatalystSerializer._ -import astraea.spark.rasterframes.experimental.datasource.awspds.L8Relation.Bands -import astraea.spark.rasterframes.expressions.transformers.{RasterSourceToRasterRefs, URIToRasterSource} -import astraea.spark.rasterframes.ref.RasterRef -import astraea.spark.rasterframes.ref.RasterSource.ReadCallback -import astraea.spark.rasterframes.rules.SpatialFilters.{Contains, Intersects} -import astraea.spark.rasterframes.rules._ -import astraea.spark.rasterframes.util._ -import com.typesafe.scalalogging.LazyLogging -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types._ -import org.apache.spark.sql.{Column, Row, SQLContext} - -/** - * Spark relation over AWS PDS Landsat 8 collection. - * - * @since 8/21/18 - */ -case class L8Relation(sqlContext: SQLContext, useTiling: Boolean, accumulator: Option[ReadCallback], filters: Seq[Filter] = Seq.empty) - extends BaseRelation with PrunedFilteredScan with SpatialRelationReceiver[L8Relation] with LazyLogging { - override def schema: StructType = L8Relation.schema - - /** Create new relation with the give filter added. */ - override def withFilter(value: Filter): L8Relation = copy(filters = filters :+ value) - - /** Check to see if relation already exists in this. */ - override def hasFilter(filter: Filter): Boolean = filters.contains(filter) - - // Most implementations don't bother with this... the hassle vs. benefit may be questionable. - override def unhandledFilters(filters: Array[Filter]): Array[Filter] = { - val TS = PDSFields.TIMESTAMP.name - super.unhandledFilters(filters).filterNot { - case GreaterThan(TS, _) ⇒ true - case GreaterThanOrEqual(TS, _) ⇒ true - case LessThan(TS, _) ⇒ true - case LessThanOrEqual(TS, _) ⇒ true - case _ ⇒ false - } - } - - // TODO: Is there a more clean, direct way of delegating filtering other - // TODO: having to reconstitute the predicates like this? - private def colExpr(filter: Filter): Column = filter match { - case GreaterThan(name, value) ⇒ col(name) > value - case GreaterThanOrEqual(name, value) ⇒ col(name) >= value - case LessThan(name, value) ⇒ col(name) < value - case LessThanOrEqual(name, value) ⇒ col(name) <= value - case EqualTo(name, value) ⇒ col(name) === value - case Intersects(name, value) ⇒ st_intersects(col(name), geomLit(value)) - case Contains(name, value) ⇒ st_contains(col(name), geomLit(value)) - } - - override def buildScan(requiredColumns: Array[String], sparkFilters: Array[Filter]): RDD[Row] = { - logger.debug(s"Required columns: ${requiredColumns.mkString(", ")}") - val aggFilters = (sparkFilters ++ splitFilters(filters)).distinct - if(aggFilters.isEmpty) { - logger.warn("No filters provided. Full catalog scan invoked.") - } - else { - logger.debug(s"Filters: $aggFilters") - } - - val catalog = sqlContext.read - .format(L8CatalogDataSource.SHORT_NAME) - .load() - .withColumnRenamed(PDSFields.ACQUISITION_DATE.name, PDSFields.TIMESTAMP.name) - .withColumn(PDSFields.BOUNDS.name, bounds_geometry(col(PDSFields.BOUNDS_WGS84.name))) - .drop(PDSFields.BOUNDS_WGS84.name) - - val filtered = aggFilters - .foldLeft(catalog)((d, filter) ⇒ d.where(colExpr(filter))) - - val (bands, other) = requiredColumns.partition(Bands.names.contains) - - val nonTile = other.map(col) - - val df = { - // NB: We assume that `nativeTiling` preserves the band names. - val expanded = RasterSourceToRasterRefs(useTiling, bands.map(b ⇒ URIToRasterSource(l8_band_url(b), accumulator).as(b)): _*) - filtered.select(nonTile :+ expanded: _*) - } - - // Make sure shape of resulting rows conforms to what was requested - val selected = requiredColumns.headOption - .map(head ⇒ df.select(head, requiredColumns.tail: _*)) - .getOrElse(df) - - selected.rdd - } -} - -object L8Relation extends PDSFields { - object Bands extends Enumeration { - type Bands = Value - val B1, B2, B3, B4, B5, B6, B7, B8, B9, B10, B11, BQA = Value - val names: Seq[String] = values.toSeq.map(_.toString) - } - - lazy val schema: StructType = { - StructType( - L8CatalogRelation.schema.collect { - case ACQUISITION_DATE ⇒ ACQUISITION_DATE.copy(name = StandardColumns.TIMESTAMP_COLUMN.columnName) - case s if s.name == BOUNDS_WGS84.name ⇒ BOUNDS - case s if s != DOWNLOAD_URL ⇒ s - } ++ L8Relation.Bands.values.toSeq.map(b ⇒ StructField(b.toString, CatalystSerializer[RasterRef].schema, true)) - ) - } -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/package.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/package.scala deleted file mode 100644 index 60b1169fc..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/package.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.datasource -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ -import astraea.spark.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._ - -/** - * Module support. - * - * @since 5/4/18 - */ -package object awspds { - /** - * Constructs link with the form: - * `https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_{bandId].TIF` - * @param bandID Band ID suffix, e.g. "B4" - * @return - */ - def l8_band_url(bandID: String): TypedColumn[Any, String] = { - concat(col("download_url"), concat(col("product_id"), lit(s"_$bandID.TIF"))) - }.as(bandID).as[String] - -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/package.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/package.scala deleted file mode 100644 index 4e6129be4..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/package.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package astraea.spark.rasterframes.experimental -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.analysis.FunctionRegistry -import org.apache.spark.sql.rf.VersionShims._ - - -/** - * Module utilitities - * - * @since 9/3/18 - */ -package object datasource { - /** Downloads the referenced URL into an uninterpreted binary data array. */ - def download(urlColumn: Column): TypedColumn[Any, Array[Byte]] = DownloadExpression(urlColumn) - - /** Downloads the contents at each of the referenced URLs, interpreting - * them as equally sized and */ - def read_tiles(urls: Column*): Column = ReadTilesExpression(urls) - - def register(sqlContext: SQLContext): Unit = { - // Expression-oriented functions have a different registration scheme - // Currently have to register with the `builtin` registry due to Spark data hiding. - val registry: FunctionRegistry = rf.registry(sqlContext) - registry.registerExpression[ReadTilesExpression]("rf_read_tiles") - } -} diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala deleted file mode 100644 index 0af839ffa..000000000 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala +++ /dev/null @@ -1,157 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2018 Astraea. Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * - */ - -package astraea.spark.rasterframes.experimental.slippy - -import java.io.PrintStream -import java.net.URI - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.util._ -import geotrellis.proj4.{LatLng, WebMercator} -import geotrellis.raster._ -import geotrellis.raster.io.geotiff._ -import geotrellis.raster.io.geotiff.tags.codes.ColorSpace -import geotrellis.raster.resample.Bilinear -import geotrellis.spark._ -import geotrellis.spark.io.slippy.HadoopSlippyTileWriter -import geotrellis.spark.pyramid.Pyramid -import geotrellis.spark.tiling.ZoomedLayoutScheme -import geotrellis.util.MethodExtensions -import org.apache.commons.lang3.text.StrSubstitutor -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.spark.annotation.Experimental - -import scala.io.Source - -/** - * Experimental support for exporting a RasterFrame into Slippy map format. - * - * @since 4/10/18 - */ -@Experimental -trait SlippyExport extends MethodExtensions[RasterFrame]{ - /** - * Export GeoTiff tiles in a slippy map directory structure; for debugging purposes only. - * NB: Temporal components are ignored blindly. - */ - @Experimental - def exportGeoTiffTiles(dest: URI): Unit = { - val spark = self.sparkSession - implicit val sc = spark.sparkContext - - val tlm = self.tileLayerMetadata.merge - val crs = tlm.crs - val mapTransform = tlm.mapTransform - - val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString, "tiff")({ (key, tile) => - val extent = mapTransform(key) - // If we have exactly 3 columns, we assume RGB color space. - val opts = GeoTiffOptions.DEFAULT - .mapWhen(_ ⇒ tile.bands.lengthCompare(3) == 0, - _.copy(colorSpace = ColorSpace.RGB) - ) - MultibandGeoTiff(tile, extent, crs, opts).toByteArray - }) - - val tlrdd: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match { - case Left(spatial) ⇒ spatial - case Right(origRDD) ⇒ - val newBounds = origRDD.metadata.bounds.flatMap[SpatialKey] { - bounds => KeyBounds(bounds.minKey.spatialKey, bounds.maxKey.spatialKey) - } - val newMD = origRDD.metadata.copy(bounds = newBounds) - val rdd = origRDD.map { case (k, v) ⇒ (k.spatialKey, v)} - ContextRDD(rdd, newMD) - } - - writer.write(0, tlrdd) - } - - /** - * Export tiles as a slippy map. For debugging purposes only. - * NB: Temporal components are ignored blindly. - * @param dest URI for Hadoop supported storage endpoint (e.g. 'file://', 'hdfs://', etc.). - * @param colorMap Optional color map to use for rendering tiles in non-RGB RasterFrames. - */ - @Experimental - def exportSlippyMap(dest: URI, renderer: MultibandRender.Profile = MultibandRender.Default): Unit = { - val spark = self.sparkSession - implicit val sc = spark.sparkContext - - val tileDirName = "rf-tiles" - - - val inputRDD: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match { - case Left(spatial) ⇒ spatial - case Right(origRDD) ⇒ - val newBounds = origRDD.metadata.bounds.flatMap[SpatialKey] { - bounds => KeyBounds(bounds.minKey.spatialKey, bounds.maxKey.spatialKey) - } - val newMD = origRDD.metadata.copy(bounds = newBounds) - val rdd = origRDD.map { case (k, v) ⇒ (k.spatialKey, v)} - ContextRDD(rdd, newMD) - } - - val layoutScheme = ZoomedLayoutScheme(WebMercator, tileSize = 256) - - val (zoom, reprojected) = inputRDD.reproject(WebMercator, layoutScheme, Bilinear) - val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString + "/" + tileDirName, "png")({ (_, tile) => - //require(tile.bandCount >= 3 || renderer.isInstanceOf[ColorRampProfile], - // "Single-band and dual-band RasterFrames require a ColorRampProfile for rendering") - val png = renderer.render(tile) - png.bytes - }) - - val center = reprojected.metadata.extent.center - - SlippyExport.writeHtml(dest, sc.hadoopConfiguration, Map( - "maxZoom" -> zoom.toString, - "id" -> tileDirName, - "viewLat" -> center.y.toString, - "viewLon" -> center.x.toString - )) - - // Pyramiding up the zoom levels, write our tiles out to the local file system. - Pyramid.upLevels(reprojected, layoutScheme, zoom, Bilinear) { (rdd, z) => - writer.write(z, rdd) - } - } -} - -object SlippyExport { - import scala.collection.JavaConverters._ - implicit class RasterFrameHasSlippy(val self: RasterFrame) extends SlippyExport - private def writeHtml(dest: URI, conf: Configuration, subs: Map[String, String]): Unit = { - val rawLines = Source.fromInputStream(getClass.getResourceAsStream("/slippy.html")).getLines() - - val subst = new StrSubstitutor(subs.asJava) - - val fs = FileSystem.get(dest, conf) - withResource(fs.create(new Path(new Path(dest), "index.html"), true)) { hout ⇒ - val out = new PrintStream(hout, true ,"UTF-8") - for(line ← rawLines) { - out.println(subst.replace(line)) - } - } - } -} - diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/CachedDatasetRelation.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/CachedDatasetRelation.scala similarity index 88% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/CachedDatasetRelation.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/CachedDatasetRelation.scala index 46dc23feb..231b2411e 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/CachedDatasetRelation.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/CachedDatasetRelation.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea, Inc. + * Copyright 2019 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -19,14 +19,14 @@ * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource import com.typesafe.scalalogging.LazyLogging -import org.apache.hadoop.fs.{FileSystem, Path ⇒ HadoopPath} +import org.apache.hadoop.fs.{FileSystem, Path => HadoopPath} import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{Dataset, Row} import org.apache.spark.sql.sources.BaseRelation -import astraea.spark.rasterframes.util._ +import org.apache.spark.sql.{Dataset, Row} +import org.locationtech.rasterframes.util._ /** * Mix-in for a data source that is cached as a parquet file. diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadSupport.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/DownloadSupport.scala similarity index 93% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadSupport.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/DownloadSupport.scala index 9d4e6e7f5..f5e0dff64 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/DownloadSupport.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/DownloadSupport.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,10 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource +package org.locationtech.rasterframes.experimental.datasource import java.io._ import java.net diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/ResourceCacheSupport.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/ResourceCacheSupport.scala similarity index 91% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/ResourceCacheSupport.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/ResourceCacheSupport.scala index 4e2e741d6..45173572f 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/ResourceCacheSupport.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/ResourceCacheSupport.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2019 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,20 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource import java.net.URI import java.time.{Duration, Instant} -import astraea.spark.rasterframes.experimental.datasource.DownloadSupport -import astraea.spark.rasterframes.util._ import com.typesafe.scalalogging.LazyLogging import org.apache.commons.io.FilenameUtils -import org.apache.hadoop.fs.{FileSystem, Path ⇒ HadoopPath} +import org.apache.hadoop.fs.{FileSystem, Path => HadoopPath} import org.apache.hadoop.io.MD5Hash +import org.locationtech.rasterframes.util._ import scala.util.Try import scala.util.control.NonFatal diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala similarity index 87% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala index 7262c2d97..aad1cd2fc 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogDataSource.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,11 +15,11 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds - +package org.locationtech.rasterframes.experimental.datasource.awspds import java.io.FileNotFoundException import java.net.URI @@ -28,6 +28,7 @@ import com.typesafe.scalalogging.LazyLogging import org.apache.hadoop.fs.FileSystem import org.apache.spark.sql.SQLContext import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationProvider} +import org.locationtech.rasterframes.experimental.datasource.ResourceCacheSupport /** * Data source for querying AWS PDS catalog of L8 imagery. @@ -48,7 +49,7 @@ class L8CatalogDataSource extends DataSourceRegister with RelationProvider { } object L8CatalogDataSource extends LazyLogging with ResourceCacheSupport { - final val SHORT_NAME: String = "awsl8-catalog" + final val SHORT_NAME: String = "aws-pds-l8-catalog" private val remoteSource = URI.create("http://landsat-pds.s3.amazonaws.com/c1/L8/scene_list.gz") private def sceneListFile(implicit fs: FileSystem) = cachedURI(remoteSource).getOrElse(throw new FileNotFoundException(remoteSource.toString)) diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala similarity index 61% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala index f5d3796b4..f87597362 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/L8CatalogRelation.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,18 +15,21 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource.awspds import com.typesafe.scalalogging.LazyLogging -import com.vividsolutions.jts.geom.Envelope -import org.apache.hadoop.fs.{Path ⇒ HadoopPath} +import geotrellis.vector.Extent +import org.apache.hadoop.fs.{Path => HadoopPath} import org.apache.spark.sql.functions._ import org.apache.spark.sql.sources.{BaseRelation, TableScan} import org.apache.spark.sql.types._ -import org.apache.spark.sql.{Dataset, Row, SQLContext} +import org.apache.spark.sql.{Dataset, Row, SQLContext, TypedColumn} +import org.locationtech.rasterframes.encoders.SparkBasicEncoders.stringEnc +import org.locationtech.rasterframes.experimental.datasource.CachedDatasetRelation /** * Schema definition and parser for AWS PDS L8 scene data. * @@ -42,9 +45,12 @@ case class L8CatalogRelation(sqlContext: SQLContext, sceneListPath: HadoopPath) protected def cacheFile: HadoopPath = sceneListPath.suffix(".parquet") protected def constructDataset: Dataset[Row] = { - import astraea.spark.rasterframes.encoders.StandardEncoders.envelopeEncoder + import org.locationtech.rasterframes.encoders.StandardEncoders.extentEncoder import sqlContext.implicits._ logger.debug("Parsing " + sceneListPath) + + val bandCols = Bands.values.toSeq.map(b => l8_band_url(b) as b.toString) + sqlContext.read .schema(inputSchema) .option("header", "true") @@ -53,21 +59,32 @@ case class L8CatalogRelation(sqlContext: SQLContext, sceneListPath: HadoopPath) .where(not($"${PRODUCT_ID.name}".endsWith("RT"))) .drop("download_url") .withColumn(BOUNDS_WGS84.name, struct( - $"min_lon" as "minX", - $"max_lon" as "maxX", - $"min_lat" as "minY", - $"max_lat" as "maxY" - ).as[Envelope]) + $"min_lon" as "xmin", + $"min_lat" as "ymin", + $"max_lon" as "xmax", + $"max_lat" as "ymax" + ).as[Extent]) .withColumnRenamed("__url", DOWNLOAD_URL.name) + .select(col("*") +: bandCols: _*) .select(schema.map(f ⇒ col(f.name)): _*) .orderBy(ACQUISITION_DATE.name, PATH.name, ROW.name) .distinct() // The scene file contains duplicates. - .repartition(8) + .repartition(8, col(PATH.name), col(ROW.name)) } } object L8CatalogRelation extends PDSFields { + /** + * Constructs link with the form: + * `https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_{bandId].TIF` + * @param band Band identifier + * @return + */ + def l8_band_url(band: Bands.Band): TypedColumn[Any, String] = { + concat(col("download_url"), concat(col("product_id"), lit(s"_$band.TIF"))) + }.as(band.toString).as[String] + private def inputSchema = StructType(Seq( PRODUCT_ID, ENTITY_ID, @@ -83,6 +100,13 @@ object L8CatalogRelation extends PDSFields { DOWNLOAD_URL )) + object Bands extends Enumeration { + type Band = Value + val B1, B2, B3, B4, B5, B6, B7, B8, B9, B10, B11, BQA = Value + val names: Seq[String] = values.toSeq.map(_.toString) + } + + def schema = StructType(Seq( PRODUCT_ID, ENTITY_ID, @@ -91,9 +115,8 @@ object L8CatalogRelation extends PDSFields { PROC_LEVEL, PATH, ROW, - BOUNDS_WGS84, - DOWNLOAD_URL - )) + BOUNDS_WGS84 + ) ++ Bands.names.map(n => StructField(n, StringType, true))) } diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala similarity index 76% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala index 84d461ad8..347d82d29 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogDataSource.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,22 +15,24 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource.awspds import java.net.URI import java.time.LocalDate import java.time.temporal.ChronoUnit -import astraea.spark.rasterframes.util.withResource -import astraea.spark.rasterframes._ +import org.locationtech.rasterframes.util.withResource +import org.locationtech.rasterframes._ import com.typesafe.scalalogging.LazyLogging -import org.apache.hadoop.fs.{FileSystem, Path ⇒ HadoopPath} +import org.apache.hadoop.fs.{FileSystem, Path => HadoopPath} import org.apache.hadoop.io.IOUtils import org.apache.spark.sql.SQLContext import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationProvider} +import org.locationtech.rasterframes.experimental.datasource.ResourceCacheSupport /** @@ -55,7 +57,7 @@ class MODISCatalogDataSource extends DataSourceRegister with RelationProvider wi require(parameters.get("path").isEmpty, "MODISCatalogDataSource doesn't support specifying a path. Please use `load()`.") sqlContext.withRasterFrames - astraea.spark.rasterframes.experimental.datasource.register(sqlContext) + org.locationtech.rasterframes.experimental.datasource.register(sqlContext) val start = parameters.get("start").map(LocalDate.parse).getOrElse(LocalDate.of(2013, 1, 1)) val end = parameters.get("end").map(LocalDate.parse).getOrElse(LocalDate.now().minusDays(7)) @@ -69,13 +71,54 @@ class MODISCatalogDataSource extends DataSourceRegister with RelationProvider wi } object MODISCatalogDataSource extends LazyLogging with ResourceCacheSupport { - final val SHORT_NAME = "aws-pds-modis" + final val SHORT_NAME = "aws-pds-modis-catalog" final val MCD43A4_BASE = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/" override def maxCacheFileAgeHours: Int = Int.MaxValue - // List of missing days + // List of missing days in PDS private val blacklist = Seq[String]( - //"2018-05-06" + "2018-02-27", + "2018-02-28", + "2018-03-01", + "2018-03-02", + "2018-03-03", + "2018-03-04", + "2018-03-05", + "2018-03-06", + "2018-03-07", + "2018-03-08", + "2018-03-09", + "2018-03-10", + "2018-03-11", + "2018-03-12", + "2018-03-13", + "2018-03-14", + "2018-03-15", + "2018-05-16", + "2018-05-17", + "2018-05-18", + "2018-05-19", + "2018-05-20", + "2018-05-21", + "2018-06-01", + "2018-06-04", + "2018-07-29", + "2018-08-03", + "2018-08-04", + "2018-08-05", + "2018-10-01", + "2018-10-02", + "2018-10-03", + "2018-10-22", + "2018-10-23", + "2018-11-12", + "2018-12-19", + "2018-12-20", + "2018-12-21", + "2018-12-22", + "2018-12-23", + "2018-12-24", + "2019-03-18" ) private def sceneFiles(start: LocalDate, end: LocalDate, useBlacklist: Boolean) = { diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala similarity index 63% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala index bb1d8e70d..8fa3422e3 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/MODISCatalogRelation.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,17 +15,20 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource.awspds import com.typesafe.scalalogging.LazyLogging -import org.apache.hadoop.fs.{Path ⇒ HadoopPath} +import org.apache.hadoop.fs.{Path => HadoopPath} import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ +import org.locationtech.rasterframes.experimental.datasource.CachedDatasetRelation +import org.locationtech.rasterframes.experimental.datasource.awspds.MODISCatalogRelation.Bands /** * Constructs a dataframe from the available scenes @@ -39,19 +42,7 @@ case class MODISCatalogRelation(sqlContext: SQLContext, sceneList: HadoopPath) protected def cacheFile: HadoopPath = sceneList.suffix(".parquet") - private val inputSchema = StructType(Seq( - StructField("date", TimestampType, false), - DOWNLOAD_URL, - GID - )) - - def schema = StructType(Seq( - PRODUCT_ID, - ACQUISITION_DATE, - GRANULE_ID, - GID, - ASSETS - )) + override def schema: StructType = MODISCatalogRelation.schema protected def constructDataset: Dataset[Row] = { import sqlContext.implicits._ @@ -61,47 +52,46 @@ case class MODISCatalogRelation(sqlContext: SQLContext, sceneList: HadoopPath) .option("header", "true") .option("mode", "DROPMALFORMED") // <--- mainly for the fact that we have internal headers from the concat .option("timestampFormat", "yyyy-MM-dd HH:mm:ss") - .schema(inputSchema) + .schema(MODISCatalogRelation.inputSchema) .csv(sceneList.toString) + val bandCols = Bands.values.toSeq.map(b => MCD43A4_band_url(b) as b.toString) + catalog .withColumn("__split_gid", split($"gid", "\\.")) .withColumn(DOWNLOAD_URL.name, regexp_replace(col(DOWNLOAD_URL.name), "index.html", "")) - .select( + .select(Seq( $"__split_gid" (0) as PRODUCT_ID.name, $"date" as ACQUISITION_DATE.name, $"__split_gid" (2) as GRANULE_ID.name, - $"${GID.name}", - MCD43A4_BAND_MAP as ASSETS.name + $"${GID.name}") ++ bandCols: _* ) .orderBy(ACQUISITION_DATE.name, GID.name) - .repartition(col(GRANULE_ID.name)) + .repartition(8, col(GRANULE_ID.name)) } } object MODISCatalogRelation extends PDSFields { - def MCD43A4_LINK(suffix: String) = - concat(col(DOWNLOAD_URL.name), concat(col(GID.name), lit(suffix))) + def MCD43A4_band_url(suffix: Bands.Band) = + concat(col(DOWNLOAD_URL.name), concat(col(GID.name), lit(s"_${suffix}.TIF"))) - val MCD43A4_BAND_MAP = map( - lit("B01"), MCD43A4_LINK("_B01.TIF"), - lit("B01qa"), MCD43A4_LINK("_B01qa.TIF"), - lit("B02"), MCD43A4_LINK("_B02.TIF"), - lit("B02qa"), MCD43A4_LINK("_B02qa.TIF"), - lit("B03"), MCD43A4_LINK("_B03.TIF"), - lit("B03qa"), MCD43A4_LINK("_B03qa.TIF"), - lit("B04"), MCD43A4_LINK("_B04.TIF"), - lit("B04qa"), MCD43A4_LINK("_B04qa.TIF"), - lit("B05"), MCD43A4_LINK("_B05.TIF"), - lit("B05qa"), MCD43A4_LINK("_B05qa.TIF"), - lit("B06"), MCD43A4_LINK("_B06.TIF"), - lit("B06qa"), MCD43A4_LINK("_B06qa.TIF"), - lit("B07"), MCD43A4_LINK("_B07.TIF"), - lit("B07qa"), MCD43A4_LINK("_B07qa.TIF"), - lit("metadata.json"), MCD43A4_LINK("_meta.json"), - lit("metadata.xml"), MCD43A4_LINK(".hdf.xml"), - lit("index.html"), concat(col(DOWNLOAD_URL.name), lit("index.html")) - ) + object Bands extends Enumeration { + type Band = Value + val B01, B01qa, B02, B02qa, B03, B03aq, B04, B04qa, B05, B05qa, B06, B06qa, B07, B07qa = Value + val names: Seq[String] = values.toSeq.map(_.toString) + } + + def schema = StructType(Seq( + PRODUCT_ID, + ACQUISITION_DATE, + GRANULE_ID, + GID + ) ++ Bands.names.map(n => StructField(n, StringType, true))) + private val inputSchema = StructType(Seq( + StructField("date", TimestampType, false), + DOWNLOAD_URL, + GID + )) } diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/PDSFields.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/PDSFields.scala similarity index 76% rename from experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/PDSFields.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/PDSFields.scala index 96eb55a7e..40f8949f7 100644 --- a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/awspds/PDSFields.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/PDSFields.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2018 Astraea. Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,14 +15,15 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 * */ -package astraea.spark.rasterframes.experimental.datasource.awspds +package org.locationtech.rasterframes.experimental.datasource.awspds -import astraea.spark.rasterframes.StandardColumns._ -import astraea.spark.rasterframes.util._ -import astraea.spark.rasterframes.encoders.StandardEncoders +import org.locationtech.rasterframes.StandardColumns._ +import org.locationtech.rasterframes.util._ +import org.locationtech.rasterframes.encoders.StandardEncoders import org.apache.spark.sql.jts.JTSTypes import org.apache.spark.sql.types._ @@ -43,10 +44,9 @@ trait PDSFields { final val PROC_LEVEL = StructField("processing_level", StringType, false) final val PATH = StructField("path", ShortType, false) final val ROW = StructField("row", ShortType, false) - final val BOUNDS = StructField(BOUNDS_COLUMN.columnName, JTSTypes.GeometryTypeInstance, false) - final val ASSETS = StructField("assets", DataTypes.createMapType(StringType, StringType)) + final val BOUNDS = StructField(GEOMETRY_COLUMN.columnName, JTSTypes.GeometryTypeInstance, false) final def BOUNDS_WGS84 = StructField( - "bounds_wgs84", StandardEncoders.envelopeEncoder.schema, false + "bounds_wgs84", StandardEncoders.extentEncoder.schema, false ) } diff --git a/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/package.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/package.scala new file mode 100644 index 000000000..bbd476528 --- /dev/null +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/awspds/package.scala @@ -0,0 +1,46 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2018 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.experimental.datasource + +import org.apache.spark.sql.DataFrameReader +import shapeless.tag +import shapeless.tag.@@ + +/** + * Module support. + * + * @since 5/4/18 + */ +package object awspds { + trait CatalogDataFrameReaderTag + type CatalogDataFrameReader = DataFrameReader @@ CatalogDataFrameReaderTag + + implicit class DataFrameReaderHasL8CatalogFormat(val reader: DataFrameReader) { + def l8Catalog: CatalogDataFrameReader = + tag[CatalogDataFrameReaderTag][DataFrameReader]( + reader.format(L8CatalogDataSource.SHORT_NAME)) + + def modisCatalog: CatalogDataFrameReader = + tag[CatalogDataFrameReaderTag][DataFrameReader]( + reader.format(MODISCatalogDataSource.SHORT_NAME)) + } +} diff --git a/core/src/test/scala/astraea/spark/rasterframes/SpatialEncodingSpec.scala b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/package.scala similarity index 57% rename from core/src/test/scala/astraea/spark/rasterframes/SpatialEncodingSpec.scala rename to experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/package.scala index 137cd9058..6c8c08aac 100644 --- a/core/src/test/scala/astraea/spark/rasterframes/SpatialEncodingSpec.scala +++ b/experimental/src/main/scala/org/locationtech/rasterframes/experimental/datasource/package.scala @@ -1,7 +1,7 @@ /* * This software is licensed under the Apache 2 license, quoted below. * - * Copyright 2017 Astraea, Inc. + * Copyright 2018 Astraea, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -15,27 +15,21 @@ * License for the specific language governing permissions and limitations under * the License. * + * SPDX-License-Identifier: Apache-2.0 + * */ -package astraea.spark.rasterframes +package org.locationtech.rasterframes.experimental + +import org.apache.spark.sql._ + /** - * Tests type encoding for spatial types. + * Module utilitities * - * @since 12/17/17 + * @since 9/3/18 */ -class SpatialEncodingSpec extends TestEnvironment with TestData { - - import sqlContext.implicits._ - - describe("Dataframe encoding ops on spatial types") { - - it("should code RDD[Point]") { - val points = Seq(null, extent.center.jtsGeom, null) - val ds = points.toDS - write(ds) - //ds.show() - assert(ds.collect().toSeq === points) - } +package object datasource { + def register(sqlContext: SQLContext): Unit = { } } diff --git a/experimental/src/test/resources/application.conf b/experimental/src/test/resources/application.conf new file mode 100644 index 000000000..36c16db32 --- /dev/null +++ b/experimental/src/test/resources/application.conf @@ -0,0 +1,8 @@ +vlm.gdal { + options { + // See https://trac.osgeo.org/gdal/wiki/ConfigOptions for options + //CPL_DEBUG = "ON" + } + // set this to `false` if CPL_DEBUG is `ON` + //useExceptions = false +} \ No newline at end of file diff --git a/experimental/src/test/scala/ndvi.sc b/experimental/src/test/scala/ndvi.sc new file mode 100644 index 000000000..483d89006 --- /dev/null +++ b/experimental/src/test/scala/ndvi.sc @@ -0,0 +1,35 @@ +import geotrellis.proj4.LatLng +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ +import org.apache.spark.sql._ +import org.apache.spark.sql.functions._ + + +implicit val spark = SparkSession.builder(). + master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames +spark.sparkContext.setLogLevel("ERROR") + +import spark.implicits._ + +val modis = spark.read.format("aws-pds-modis-catalog").load() + +val red_nir_monthly_2017 = modis + .select($"granule_id", month($"acquisition_date") as "month", $"B01" as "red", $"B02" as "nir") + .where(year($"acquisition_date") === 2017 && (dayofmonth($"acquisition_date") === 15) && $"granule_id" === "h21v09") + +val red_nir_tiles_monthly_2017 = spark.read.raster + .fromCatalog(red_nir_monthly_2017, "red", "nir") + .load() + +val result = red_nir_tiles_monthly_2017 + .where(st_intersects( + st_reproject(rf_geometry($"red"), rf_crs($"red"), LatLng), + st_makePoint(34.870605, -4.729727) + )) + .groupBy("month") + .agg(rf_agg_stats(rf_normalized_difference($"nir", $"red")) as "ndvi_stats") + .orderBy("month") + .select("month", "ndvi_stats.*") + + +result.show() \ No newline at end of file diff --git a/project/BenchmarkPlugin.scala b/project/BenchmarkPlugin.scala index f795fe536..e4f1d1acc 100644 --- a/project/BenchmarkPlugin.scala +++ b/project/BenchmarkPlugin.scala @@ -70,7 +70,7 @@ object BenchmarkPlugin extends AutoPlugin { val pat = (".*" + file + ".*").r jmhRun(pat) }.evaluated, - libraryDependencies += "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.19" + libraryDependencies += "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.21" ) def jmhRun(filePattern: Regex) = Def.taskDyn { @@ -94,7 +94,7 @@ object BenchmarkPlugin extends AutoPlugin { (run in Jmh).toTask(args) } - val benchFilesParser: Def.Initialize[State => Parser[File]] = Def.setting { (state: State) => + val benchFilesParser: Def.Initialize[State => Parser[File]] = Def.setting { state: State => val extracted = Project.extract(state) val pat = new PatternFilter( extracted.getOpt(jmhFileRegex).getOrElse(".*".r).pattern diff --git a/project/ProjectPlugin.scala b/project/ProjectPlugin.scala deleted file mode 100644 index 4f575edb2..000000000 --- a/project/ProjectPlugin.scala +++ /dev/null @@ -1,170 +0,0 @@ -import sbt.Keys._ -import sbt._ -import sbtassembly.AssemblyKeys.assembly -import sbtassembly.AssemblyPlugin.autoImport._ -import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ -import sbtrelease.ReleasePlugin.autoImport._ - -import com.servicerocket.sbt.release.git.flow.Steps._ -import com.typesafe.sbt.sbtghpages.GhpagesPlugin.autoImport._ -import com.typesafe.sbt.site.SitePlugin.autoImport._ -import xerial.sbt.Sonatype.autoImport._ - -/** - * @since 8/20/17 - */ -object ProjectPlugin extends AutoPlugin { - override def trigger: PluginTrigger = allRequirements - import autoImport._ - - override def projectSettings = Seq( - organization := "org.locationtech.rasterframes", - //organization := "io.astraea", - organizationName := "LocationTech RasterFrames", - startYear := Some(2017), - homepage := Some(url("http://rasterframes.io")), - scmInfo := Some(ScmInfo(url("https://github.com/locationtech/rasterframes"), "git@github.com:locationtech/rasterframes.git")), - description := "RasterFrames brings the power of Spark DataFrames to geospatial raster data, empowered by the map algebra and tile layer operations of GeoTrellis", - licenses += ("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0.html")), - scalaVersion := "2.11.12", - scalacOptions ++= Seq("-feature", "-deprecation"), - scalacOptions in (Compile, doc) ++= Seq("-no-link-warnings"), - javacOptions ++= Seq("-source", "1.8", "-target", "1.8"), - cancelable in Global := true, - resolvers ++= Seq( - "locationtech-releases" at "https://repo.locationtech.org/content/groups/releases", - "boundless-releases" at "https://repo.boundlessgeo.com/main/", - "Open Source Geospatial Foundation Repository" at "http://download.osgeo.org/webdav/geotools/" - ), - // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py - rfSparkVersion in ThisBuild := "2.3.2" , - rfGeoTrellisVersion in ThisBuild := "2.1.0", - rfGeoMesaVersion in ThisBuild := "2.1.0", - publishTo := sonatypePublishTo.value, - publishMavenStyle := true, - publishArtifact in (Compile, packageDoc) := true, - publishArtifact in Test := false, - fork in Test := true, - javaOptions in Test := Seq("-Xmx2G", "-Djava.library.path=/usr/local/lib"), - parallelExecution in Test := false, - testOptions in Test += Tests.Argument("-oDF"), - developers := List( - Developer( - id = "metasim", - name = "Simeon H.K. Fitch", - email = "fitch@astraea.io", - url = url("http://www.astraea.io") - ), - Developer( - id = "mteldridge", - name = "Matt Eldridge", - email = "meldridge@astraea.io", - url = url("http://www.astraea.io") - ), - Developer( - id = "bguseman", - name = "Ben Guseman", - email = "bguseman@astraea.io", - url = url("http://www.astraea.io") - ), - Developer( - id = "vpipkt", - name = "Jason Brown", - email = "jbrown@astraea.io", - url = url("http://www.astraea.io") - ) - ), - initialCommands in console := - """ - |import org.apache.spark._ - |import org.apache.spark.sql._ - |import org.apache.spark.sql.functions._ - |import geotrellis.raster._ - |import geotrellis.spark._ - |import astraea.spark.rasterframes._ - |implicit val spark = SparkSession.builder().master("local[*]").getOrCreate().withRasterFrames - """.stripMargin.trim, - cleanupCommands in console := "spark.stop()" - ) - - object autoImport { - val rfSparkVersion = settingKey[String]("Apache Spark version") - val rfGeoTrellisVersion = settingKey[String]("GeoTrellis version") - val rfGeoMesaVersion = settingKey[String]("GeoMesa version") - - def geotrellis(module: String) = Def.setting { - "org.locationtech.geotrellis" %% s"geotrellis-$module" % rfGeoTrellisVersion.value - } - def spark(module: String) = Def.setting { - "org.apache.spark" %% s"spark-$module" % rfSparkVersion.value - } - - val scalaTest = "org.scalatest" %% "scalatest" % "3.0.3" % Test - - def assemblySettings: Seq[Def.Setting[_]] = Seq( - test in assembly := {}, - assemblyMergeStrategy in assembly := { - case "logback.xml" ⇒ MergeStrategy.singleOrError - case "git.properties" ⇒ MergeStrategy.discard - case x if Assembly.isConfigFile(x) ⇒ MergeStrategy.concat - case PathList(ps @ _*) if Assembly.isReadme(ps.last) || Assembly.isLicenseFile(ps.last) ⇒ - MergeStrategy.rename - case PathList("META-INF", xs @ _*) ⇒ - xs map {_.toLowerCase} match { - case "manifest.mf" :: Nil | "index.list" :: Nil | "dependencies" :: Nil ⇒ - MergeStrategy.discard - case ps @ _ :: _ if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") ⇒ - MergeStrategy.discard - case "plexus" :: _ ⇒ - MergeStrategy.discard - case "services" :: _ ⇒ - MergeStrategy.filterDistinctLines - case "spring.schemas" :: Nil | "spring.handlers" :: Nil ⇒ - MergeStrategy.filterDistinctLines - case "maven" :: rest if rest.lastOption.exists(_.startsWith("pom")) ⇒ - MergeStrategy.discard - case _ ⇒ MergeStrategy.deduplicate - } - - case _ ⇒ MergeStrategy.deduplicate - } - ) - - def releaseSettings: Seq[Def.Setting[_]] = { - val buildSite: State ⇒ State = releaseStepTask(makeSite in LocalProject("docs")) - val publishSite: State ⇒ State = releaseStepTask(ghpagesPushSite in LocalProject("docs")) - Seq( - releaseIgnoreUntrackedFiles := true, - releaseTagName := s"${version.value}", - releaseProcess := Seq[ReleaseStep]( - checkSnapshotDependencies, - checkGitFlowExists, - inquireVersions, - runTest, - gitFlowReleaseStart, - setReleaseVersion, - buildSite, - publishSite, - commitReleaseVersion, - tagRelease, - releaseStepCommand("publishSigned"), - releaseStepCommand("sonatypeReleaseAll"), - gitFlowReleaseFinish, - setNextVersion, - commitNextVersion - ), - commands += Command.command("bumpVersion"){ st ⇒ - val extracted = Project.extract(st) - val ver = extracted.get(version) - val nextFun = extracted.runTask(releaseNextVersion, st)._2 - - val nextVersion = nextFun(ver) - - val file = extracted.get(releaseVersionFile) - IO.writeLines(file, Seq(s"""version in ThisBuild := "$nextVersion"""")) - extracted.appendWithSession(Seq(version := nextVersion), st) - } - ) - } - } -} diff --git a/project/PythonBuildPlugin.scala b/project/PythonBuildPlugin.scala new file mode 100644 index 000000000..7cc33fda3 --- /dev/null +++ b/project/PythonBuildPlugin.scala @@ -0,0 +1,181 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +import sbt.Keys.{`package`, _} +import sbt._ +import complete.DefaultParsers._ +import sbt.Tests.Summary +import sbt.util.CacheStore + +import scala.sys.process.Process +import sbtassembly.AssemblyPlugin.autoImport.assembly + +object PythonBuildPlugin extends AutoPlugin { + override def trigger: PluginTrigger = allRequirements + override def requires = RFAssemblyPlugin + + object autoImport { + val Python = config("python") + val pythonCommand = settingKey[String]("Python command. Defaults to 'python'") + val pySetup = inputKey[Int]("Run 'python setup.py '. Returns exit code.") + val pyWhl = taskKey[File]("Builds the Python wheel distribution") + } + import autoImport._ + + val copyPySources = Def.task { + val log = streams.value.log + val destDir = (Python / target).value + val cacheDir = streams.value.cacheDirectory + val maps = (Python / mappings).value + val resolved = maps map { case (file, d) => (file, destDir / d) } + log.info(s"Synchronizing ${maps.size} files to '${destDir}'") + Sync.sync(CacheStore(cacheDir / "python"))(resolved) + destDir + } + + val pyWhlJar = Def.task { + val log = streams.value.log + val buildDir = (Python / target).value + val asmbl = (Compile / assembly).value + val dest = buildDir / "deps" / "jars" / asmbl.getName + IO.copyFile(asmbl, dest) + log.info(s"PyRasterFrames assembly written to '$dest'") + dest + }.dependsOn(copyPySources) + + val pyWhlImp = Def.task { + val log = streams.value.log + val buildDir = (Python / target).value + val retcode = pySetup.toTask(" build bdist_wheel").value + if(retcode != 0) throw new RuntimeException(s"'python setup.py' returned $retcode") + val whls = (buildDir / "dist" ** "pyrasterframes*.whl").get() + require(whls.length == 1, "Running setup.py should have produced a single .whl file. Try running `clean` first.") + log.info(s"Python .whl file written to '${whls.head}'") + whls.head + }.dependsOn(pyWhlJar) + + val pyWhlAsZip = Def.task { + val log = streams.value.log + val pyDest = (packageBin / artifactPath).value + val whl = pyWhl.value + IO.copyFile(whl, pyDest) + log.info(s"Maven Python artifact written to '$pyDest'") + pyDest + }.dependsOn(pyWhl) + + override def projectConfigurations: Seq[Configuration] = Seq(Python) + + override def projectSettings = Seq( + assembly / test := {}, + pythonCommand := "python", + pySetup := { + val s = streams.value + val wd = copyPySources.value + val args = spaceDelimited("").parsed + val cmd = Seq(pythonCommand.value, "setup.py") ++ args + val ver = version.value + s.log.info(s"Running '${cmd.mkString(" ")}' in '$wd'") + val ec = Process(cmd, wd, "RASTERFRAMES_VERSION" -> ver).! + if (ec != 0) + throw new MessageOnlyException(s"'${cmd.mkString(" ")}' exited with value '$ec'") + ec + }, + pyWhl := pyWhlImp.value, + Compile / `package` := (Compile / `package`).dependsOn(Python / packageBin).value, + Test / testQuick := (Python / testQuick).evaluated, + Test / executeTests := { + val standard = (Test / executeTests).value + standard.overall match { + case TestResult.Passed => + (Python / executeTests).value + case _ ⇒ + val pySummary = Summary("pyrasterframes", "tests skipped due to scalatest failures") + standard.copy(summaries = standard.summaries ++ Iterable(pySummary)) + } + } + ) ++ + inConfig(Python)(Seq( + sourceDirectory := (Compile / sourceDirectory).value / "python", + sourceDirectories := Seq((Python / sourceDirectory).value), + target := (Compile / target).value / "python", + includeFilter := "*", + excludeFilter := HiddenFileFilter || "__pycache__" || "*.egg-info", + sources := Defaults.collectFiles(Python / sourceDirectories, Python / includeFilter, Python / excludeFilter).value, + mappings := Defaults.relativeMappings(Python / sources, Python / sourceDirectories).value, + packageBin := Def.sequential( + Compile / packageBin, + pyWhl, + pyWhlAsZip, + ).value, + packageBin / artifact := { + val java = (Compile / packageBin / artifact).value + java.withType("zip").withClassifier(Some("python")).withExtension("zip") + }, + packageBin / artifactPath := { + val dest = (Compile / packageBin / artifactPath).value.getParentFile + val art = (Python / packageBin / artifact).value + val ver = version.value + dest / s"${art.name}-$ver-py2.py3-none-any.whl" + }, + testQuick := pySetup.toTask(" test").value, + executeTests := Def.task { + val resultCode = pySetup.toTask(" test").value + val msg = resultCode match { + case 1 ⇒ "There are Python test failures." + case 2 ⇒ "Python test execution was interrupted." + case 3 ⇒ "Internal error during Python test execution." + case 4 ⇒ "PyTest usage error." + case 5 ⇒ "No Python tests found." + case x if x != 0 ⇒ "Unknown error while running Python tests." + case _ ⇒ "PyRasterFrames tests successfully completed." + } + val pySummary = Summary("pyrasterframes", msg) + // Would be cool to derive this from the python output... + val result = if (resultCode == 0) { + new SuiteResult( + TestResult.Passed, + passedCount = 1, + failureCount = 0, + errorCount = 0, + skippedCount = 0, + ignoredCount = 0, + canceledCount = 0, + pendingCount = 0 + ) + } + else { + new SuiteResult( + TestResult.Failed, + passedCount = 0, + failureCount = 1, + errorCount = 0, + skippedCount = 0, + ignoredCount = 0, + canceledCount = 0, + pendingCount = 0 + ) + } + result + Tests.Output(result.result, Map("Python Tests" -> result), Iterable(pySummary)) + }.dependsOn(assembly).value + )) ++ + addArtifact(Python / packageBin / artifact, Python / packageBin) +} diff --git a/project/RFAssemblyPlugin.scala b/project/RFAssemblyPlugin.scala new file mode 100644 index 000000000..06246d3ae --- /dev/null +++ b/project/RFAssemblyPlugin.scala @@ -0,0 +1,98 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +import sbt._ +import sbt.Keys._ +import sbtassembly.AssemblyPlugin +import sbtassembly.AssemblyPlugin.autoImport.{ShadeRule, _} + +import scala.util.matching.Regex + +/** + * Standard support for creating assembly jars. + */ +object RFAssemblyPlugin extends AutoPlugin { + override def requires = AssemblyPlugin + + implicit class RichRegex(val self: Regex) extends AnyVal { + def =~(s: String) = self.pattern.matcher(s).matches + } + + object autoImport { + val assemblyExcludedJarPatterns = settingKey[Seq[Regex]]( + "List of regular expressions identifying jar file names that will be force-excluded from assembly" + ) + } + + override def projectSettings = Seq( + test in assembly := {}, + autoImport.assemblyExcludedJarPatterns := Seq( + "scalatest.*".r, + "junit.*".r + ), + assemblyShadeRules in assembly := { + val shadePrefixes = Seq( + "shapeless", + "com.amazonaws", + "org.apache.avro", + "org.apache.http", + "com.google.guava" + ) + shadePrefixes.map(p ⇒ ShadeRule.rename(s"$p.**" -> s"rf.shaded.$p.@1").inAll) + }, + assemblyOption in assembly := + (assemblyOption in assembly).value.copy(includeScala = false), + assemblyJarName in assembly := s"${normalizedName.value}-assembly-${version.value}.jar", + assemblyExcludedJars in assembly := { + val cp = (fullClasspath in assembly).value + val excludedJarPatterns = autoImport.assemblyExcludedJarPatterns.value + cp filter { jar ⇒ + excludedJarPatterns + .exists(_ =~ jar.data.getName) + } + }, + assemblyMergeStrategy in assembly := { + case "logback.xml" ⇒ MergeStrategy.singleOrError + case "git.properties" ⇒ MergeStrategy.discard + case x if Assembly.isConfigFile(x) ⇒ MergeStrategy.concat + case PathList(ps @ _*) if Assembly.isReadme(ps.last) || Assembly.isLicenseFile(ps.last) ⇒ + MergeStrategy.rename + case PathList("META-INF", xs @ _*) ⇒ + xs map { _.toLowerCase } match { + case "manifest.mf" :: Nil | "index.list" :: Nil | "dependencies" :: Nil ⇒ + MergeStrategy.discard + case ps @ x :: _ if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") ⇒ + MergeStrategy.discard + case "plexus" :: _ ⇒ + MergeStrategy.discard + case "services" :: _ ⇒ + MergeStrategy.filterDistinctLines + case "spring.schemas" :: Nil | "spring.handlers" :: Nil ⇒ + MergeStrategy.filterDistinctLines + case "maven" :: rest if rest.lastOption.exists(_.startsWith("pom")) ⇒ + MergeStrategy.discard + case _ ⇒ MergeStrategy.deduplicate + } + + case _ ⇒ MergeStrategy.deduplicate + } + ) +} \ No newline at end of file diff --git a/project/RFDependenciesPlugin.scala b/project/RFDependenciesPlugin.scala new file mode 100644 index 000000000..85fa87923 --- /dev/null +++ b/project/RFDependenciesPlugin.scala @@ -0,0 +1,67 @@ + +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +import sbt.Keys._ +import sbt._ + +object RFDependenciesPlugin extends AutoPlugin { + override def trigger: PluginTrigger = allRequirements + object autoImport { + val rfSparkVersion = settingKey[String]("Apache Spark version") + val rfGeoTrellisVersion = settingKey[String]("GeoTrellis version") + val rfGeoMesaVersion = settingKey[String]("GeoMesa version") + + def geotrellis(module: String) = Def.setting { + "org.locationtech.geotrellis" %% s"geotrellis-$module" % rfGeoTrellisVersion.value + } + def spark(module: String) = Def.setting { + "org.apache.spark" %% s"spark-$module" % rfSparkVersion.value + } + def geomesa(module: String) = Def.setting { + "org.locationtech.geomesa" %% s"geomesa-$module" % rfGeoMesaVersion.value + } + + val scalatest = "org.scalatest" %% "scalatest" % "3.0.3" % Test + val shapeless = "com.chuusai" %% "shapeless" % "2.3.2" + val `jts-core` = "org.locationtech.jts" % "jts-core" % "1.16.0" + val `geotrellis-contrib-vlm` = "com.azavea.geotrellis" %% "geotrellis-contrib-vlm" % "2.12.0" + val `geotrellis-contrib-gdal` = "com.azavea.geotrellis" %% "geotrellis-contrib-gdal" % "2.12.0" + + val scaffeine = "com.github.blemale" %% "scaffeine" % "2.6.0" + } + import autoImport._ + + override def projectSettings = Seq( + resolvers ++= Seq( + "locationtech-releases" at "https://repo.locationtech.org/content/groups/releases", + "Azavea Public Builds" at "https://dl.bintray.com/azavea/geotrellis", + "boundless-releases" at "https://repo.boundlessgeo.com/main/", + "Open Source Geospatial Foundation Repository" at "http://download.osgeo.org/webdav/geotools/" + ), + + // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py + rfSparkVersion := "2.3.3", + rfGeoTrellisVersion := "2.3.1", + rfGeoMesaVersion := "2.2.1", + //dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.58d4965" + ) +} diff --git a/project/RFProjectPlugin.scala b/project/RFProjectPlugin.scala new file mode 100644 index 000000000..b7c904416 --- /dev/null +++ b/project/RFProjectPlugin.scala @@ -0,0 +1,86 @@ +import com.typesafe.sbt.{GitPlugin, SbtGit} +import com.typesafe.sbt.SbtGit.git +import sbt.Keys._ +import sbt._ +import xerial.sbt.Sonatype.autoImport._ + +/** + * @since 8/20/17 + */ +object RFProjectPlugin extends AutoPlugin { + override def trigger: PluginTrigger = allRequirements + override def requires = GitPlugin + + override def projectSettings = Seq( + organization := "org.locationtech.rasterframes", + organizationName := "LocationTech RasterFrames", + startYear := Some(2017), + homepage := Some(url("http://rasterframes.io")), + git.remoteRepo := "git@github.com:locationtech/rasterframes.git", + scmInfo := Some(ScmInfo(url("https://github.com/locationtech/rasterframes"), "git@github.com:locationtech/rasterframes.git")), + description := "RasterFrames brings the power of Spark DataFrames to geospatial raster data.", + licenses += ("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0.html")), + scalaVersion := "2.11.12", + scalacOptions ++= Seq( + "-feature", + "-deprecation", + "-Ywarn-dead-code", + "-Ywarn-unused-import" + ), + scalacOptions in (Compile, doc) ++= Seq("-no-link-warnings"), + console / scalacOptions := Seq("-feature"), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8"), + cancelable in Global := true, + publishTo in ThisBuild := sonatypePublishTo.value, + publishMavenStyle := true, + publishArtifact in (Compile, packageDoc) := true, + publishArtifact in Test := false, + fork in Test := true, + javaOptions in Test := Seq("-Xmx2G", "-Djava.library.path=/usr/local/lib"), + parallelExecution in Test := false, + testOptions in Test += Tests.Argument("-oDF"), + developers := List( + Developer( + id = "metasim", + name = "Simeon H.K. Fitch", + email = "fitch@astraea.earth", + url = url("http://www.astraea.earth") + ), + Developer( + id = "mteldridge", + name = "Matt Eldridge", + email = "meldridge@astraea.earth", + url = url("http://www.astraea.earth") + ), + Developer( + id = "bguseman", + name = "Ben Guseman", + email = "bguseman@astraea.earth", + url = url("http://www.astraea.earth") + ), + Developer( + id = "vpipkt", + name = "Jason Brown", + email = "jbrown@astraea.earth", + url = url("http://www.astraea.earth") + ) + ), + initialCommands in console := + """ + |import org.apache.spark._ + |import org.apache.spark.sql._ + |import org.apache.spark.sql.functions._ + |import geotrellis.raster._ + |import geotrellis.spark._ + |import org.locationtech.rasterframes._ + |implicit val spark = SparkSession.builder() + | .master("local[*]") + | .withKryoSerialization + | .getOrCreate() + | .withRasterFrames + |spark.sparkContext.setLogLevel("ERROR") + |import spark.implicits._ + """.stripMargin.trim, + cleanupCommands in console := "spark.stop()" + ) +} diff --git a/project/RFReleasePlugin.scala b/project/RFReleasePlugin.scala new file mode 100644 index 000000000..7eef23231 --- /dev/null +++ b/project/RFReleasePlugin.scala @@ -0,0 +1,107 @@ + +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +import sbt.Keys._ +import sbt._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ +import sbtrelease.ReleasePlugin.autoImport._ +import com.typesafe.sbt.sbtghpages.GhpagesPlugin +import com.typesafe.sbt.sbtghpages.GhpagesPlugin.autoImport.ghpagesPushSite +import com.typesafe.sbt.site.SitePlugin +import com.typesafe.sbt.site.SitePlugin.autoImport.makeSite +import scala.sys.process.{Process => SProcess} + +/** Release process support. */ +object RFReleasePlugin extends AutoPlugin { + override def trigger: PluginTrigger = noTrigger + override def requires = RFProjectPlugin && SitePlugin && GhpagesPlugin + override def projectSettings = { + val buildSite: State ⇒ State = releaseStepTask(makeSite in LocalProject("docs")) + val publishSite: State ⇒ State = releaseStepTask(ghpagesPushSite in LocalProject("docs")) + Seq( + releaseIgnoreUntrackedFiles := true, + releaseTagName := s"${version.value}", + releaseProcess := Seq[ReleaseStep]( + checkSnapshotDependencies, + checkGitFlowExists, + inquireVersions, + runClean, + runTest, + gitFlowReleaseStart, + setReleaseVersion, + buildSite, + commitReleaseVersion, + tagRelease, + releaseStepCommand("publishSigned"), + releaseStepCommand("sonatypeReleaseAll"), + publishSite, + gitFlowReleaseFinish, + setNextVersion, + commitNextVersion, + remindMeToPush + ), + commands += Command.command("bumpVersion"){ st ⇒ + val extracted = Project.extract(st) + val ver = extracted.get(version) + val nextFun = extracted.runTask(releaseNextVersion, st)._2 + + val nextVersion = nextFun(ver) + + val file = extracted.get(releaseVersionFile) + IO.writeLines(file, Seq(s"""version in ThisBuild := "$nextVersion"""")) + extracted.appendWithSession(Seq(version := nextVersion), st) + } + ) + } + + def releaseVersion(state: State): String = + state.get(ReleaseKeys.versions).map(_._1).getOrElse { + sys.error("No versions are set! Was this release part executed before inquireVersions?") + } + + val gitFlowReleaseStart = ReleaseStep(state ⇒ { + val version = releaseVersion(state) + SProcess(Seq("git", "flow", "release", "start", version)).! + state + }) + + val gitFlowReleaseFinish = ReleaseStep(state ⇒ { + val version = releaseVersion(state) + SProcess(Seq("git", "flow", "release", "finish", "-n", s"$version")).! + state + }) + + val remindMeToPush = ReleaseStep(state ⇒ { + state.log.warn("Don't forget to git push master AND develop!") + state + }) + + val checkGitFlowExists = ReleaseStep(state => { + SProcess(Seq("command", "-v", "git-flow")).!! match { + case "" => sys.error("git-flow is required for release. See https://github.com/nvie/gitflow for installation instructions.") + case _ => SProcess(Seq("git", "flow", "init", "-d")).! match { + case 0 => state + case e => sys.error(s"git-flow init failed with error code $e") + } + } + }) +} diff --git a/project/plugins.sbt b/project/plugins.sbt index a3fd9b0e9..5835c4f93 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,30 +1,18 @@ logLevel := sbt.Level.Error -resolvers += Resolver.bintrayIvyRepo("s22s", "sbt-plugins") - -addSbtPlugin("com.servicerocket" % "sbt-git-flow" % "0.1.3-astraea.1") - +addSbtCoursier addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") - addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0") - addSbtPlugin("de.heikoseeberger" % "sbt-header" % "3.0.2") - -addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.6.10") - addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.2") - addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.2") -addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.5.0") - -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.4") - +addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.5.5") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.6") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.1") - addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.1") - -addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.7-astraea.1") - addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "0.2.10") +addSbtPlugin("com.github.gseitz" %% "sbt-release" % "1.0.9") +addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19") + diff --git a/project/project/plugins.sbt b/project/project/plugins.sbt new file mode 100644 index 000000000..9c5ecac47 --- /dev/null +++ b/project/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.1.0-M11") \ No newline at end of file diff --git a/pyrasterframes/README.md b/pyrasterframes/README.md new file mode 100644 index 000000000..5c4096a80 --- /dev/null +++ b/pyrasterframes/README.md @@ -0,0 +1,165 @@ +# PyRasterFrames + +## Using PyRasterFrames + +See [here](src/main/python/README.md) for user facing details. + +## Deployment + +The pip installation focuses on local mode operation of spark. To deploy spark applications you will need to consider a few configurations. + +### `SparkSession` Setup + +#### Python shell + + +To initialize RasterFrames in a generic Python shell, use the provided convenience function as follows: + +```python +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import * +spark = create_rf_spark_session() +``` + +If you require further customization you can follow the general pattern below: + +```python +from pyspark.sql import SparkSession +from pyrasterframes.utils import find_pyrasterframes_assembly +from pyrasterframes.rasterfunctions import * +rf_jar = find_pyrasterframes_assembly() +spark = (SparkSession.builder + .master("local[*]") + .appName("RasterFrames") + .config("spark.jars", rf_jar) + .config("spark.some.config.option", "some-value") + .withKryoSerialization() + .getOrCreate()) + .withRasterFrames() +``` + +#### Pyspark shell or app + +To quickly get the command to run a `pyspark` shell with PyRasterFrames enabled, run + +```bash +sbt pySparkCmd + +# > PYTHONSTARTUP=/var/somewhere/pyrf_init.py pyspark --jars /pyrasterframes/target/scala-2.11/pyrasterframes-assembly-${VERSION}.jar --py-files /pyrasterframes/target/scala-2.11/pyrasterframes-python-${VERSION}.zip +``` + +The runtime dependencies will be created and the command to run printed to the console. + +To manually initialize PyRasterFrames in a `pyspark` shell, prepare to call pyspark with the appropriate `--master` and other `--conf` arguments for your cluster manager and environment. To these you will add the PyRasterFrames assembly JAR and the python source zip. See below for how to build or download those artifacts. + +```bash + pyspark \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ + --conf spark.kryo.registrator=org.locationtech.rasterframes.util.RFKryoRegistrator \ + --conf spark.kryoserializer.buffer.max=500m \ + --jars pyrasterframes/target/scala-2.11/pyrasterframes-assembly-${VERSION}.jar \ + --py-files pyrasterframes/target/scala-2.11/pyrasterframes-python-${VERSION}.zip + +``` + +Then in the PySpark shell or script, import the module and call `withRasterFrames` on the SparkSession. + +```python +from pyrasterframes.utils import create_rf_spark_session +spark = create_rf_spark_session() +df = spark.read.raster('https://landsat-pds.s3.amazonaws.com/c1/L8/158/072/LC08_L1TP_158072_20180515_20180604_01_T1/LC08_L1TP_158072_20180515_20180604_01_T1_B5.TIF') +``` + +## Development + +RasterFrames is primarily implemented in Scala, and as such uses the Scala build tool [`sbt`](https://www.scala-sbt.org/). +All `sbt` commands referenced below must be run from the root source directory, i.e. the parent of the `pyrasterframes` +directory, including Python-related build steps. + +As a tip, know that `sbt` is much faster if run in "interactive" mode, where you launch `sbt` with no arguments, +and subsequent commands are invoked via an interactive shell. But for context clarity, we'll prefix each command +example below with `sbt`. + + +## Running Tests + +The PyRasterFrames unit tests can found in `/pyrasterframes/python/tests`. To run them: + +```bash +sbt pyrasterframes/test # alias 'pyTest' +``` + +*See also the below discussion of running `setup.py` for more options to run unit tests.* + +## Running Python Markdown Sources + +The markdown documentation in `/pyrasterframes/src/main/python/docs` contains code blocks that are evaluated by the build to show results alongside examples. The processed markdown source can be found in `/pyrasterframes/target/python/docs`. + +```bash +sbt pyrasterframes/doc # alias 'pyDoc' +``` + +See [`docs/README.md`](../docs/README.md) for details. + + +## Creating and Using a Build + +Assuming that `$SCALA_VER` is the major verison of Scala in use (e.g. 2.11) , and `$VER` is the version of RasterFrames, +the primary build artifacts are: + +* JVM library: `/pyrasterframes/target/scala-$SCALA_VER/pyrasterframes_$SCALA_VER-$VER.jar` +* Python .whl package with assembly: `/pyrasterframes/target/python/dist/pyrasterframes-$VER-py2.py3-none-any.whl` +* Python package as Maven .zip artifact: `/pyrasterframes/target/scala-$SCALA_VER/pyrasterframes-python-$VER.zip` + +You build them with: + +```bash +sbt pyrasterframes/package # alias 'pyBuild' +``` + +Release versions of these artifacts are published to https://central.sonatype.org/ under the Maven/Ivy groupId:artifactId:version (GAV) coordinates +`org.locationtech.rasterframes:pyrasterframes_$SCALA_VER:$VER`. + +Latest version can be found [here](https://search.maven.org/search?q=g:org.locationtech.rasterframes). +The Python package is published under the `python` classifier, `zip` extension. + +### Building and publishing for pypi + +To create a source and binary distribution appropriate for pypi: + +```bash +sbt pyrasterframes/package # alias 'pyBuild' +``` + +Observe the output messages such as: + + [info] Python .whl file written to '/Users/monty/rasterframes/pyrasterframes/target/python/dist/pyrasterframes-${VERSION}-py2.py3-none-any.whl' + +This wheel is suitable for publishing to a package index. See [packaging.python.org](https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives). + +## Build Internals + +### Running `setup.py` + +Before a build is initiated, the Python sources are copied to `pyrasterframes/target/python`. This ensures the +version controlled source directories are not polluted by `setuptools` residuals, better ensuring repeatable builds. To +simplify the process of working with `setup.py` in this context, a `pySetup` sbt interactive command is available. To +illustrate its usage, suppose we want to run a subset of the Python unit test. Instead of running `sbt pyTests` you can: + +```bash +sbt 'pySetup test --addopts "-k test_tile_creation"' +``` + +Or to build a specific document: + +```bash +sbt 'pySetup pweave -s docs/raster-io.pymd' +``` + +Or to build a specific document with desired output format: + +```bash +sbt 'pySetup pweave -f notebook -s docs/numpy-pandas.pymd' +``` + +*Note: You may need to run `sbt pyrasterframes/package` and/or `sbt pyTest` at least once for certain `pySetup` commands to work.* diff --git a/pyrasterframes/build.sbt b/pyrasterframes/build.sbt index c4284836e..707f6ce14 100644 --- a/pyrasterframes/build.sbt +++ b/pyrasterframes/build.sbt @@ -1,222 +1,42 @@ -import scala.sys.process.Process -import sbt.Keys.`package` -import sbt.Tests.Summary +addCommandAlias("pyDocs", "pyrasterframes/doc") +addCommandAlias("pyTest", "pyrasterframes/test") +addCommandAlias("pyBuild", "pyrasterframes/package") -enablePlugins(SparkPackagePlugin, AssemblyPlugin) +exportJars := true +Python / doc / sourceDirectory := (Python / target).value / "docs" +Python / doc / target := (Python / target).value / "docs" +Python / doc := (Python / doc / target).toTask.dependsOn( + Def.sequential( + assembly, + Test / compile, + pySetup.toTask(" pweave") + ) +).value -lazy val assemblyHasGeoTrellis = settingKey[Boolean]("If false, GeoTrellis libraries are excluded from assembly.") -assemblyHasGeoTrellis := true +doc := (Python / doc).value -assembly / assemblyExcludedJars ++= { - val cp = (fullClasspath in assembly).value - val gt = assemblyHasGeoTrellis.value - if(gt) Seq.empty - else cp.filter(_.data.getName.contains("geotrellis")) -} - -assembly / test := {} - -//-------------------------------------------------------------------- -// Spark Packages Plugin -//-------------------------------------------------------------------- - -spName := "io.astraea/pyrasterframes" -sparkVersion := rfSparkVersion.value -sparkComponents ++= Seq("sql", "mllib") -spAppendScalaVersion := false -spIncludeMaven := false -spIgnoreProvided := true -spShade := true -spPackage := { - val dist = spDist.value - val extracted = IO.unzip(dist, (target in spPackage).value, GlobFilter("*.jar")) - if(extracted.size != 1) sys.error("Didn't expect to find multiple .jar files in distribution.") - extracted.head -} -spShortDescription := description.value -spHomepage := homepage.value.get.toString -spDescription := """ - |RasterFrames brings the power of Spark DataFrames to geospatial raster data, - |empowered by the map algebra and tile layer operations of GeoTrellis. - | - |The underlying purpose of RasterFrames is to allow data scientists and software - |developers to process and analyze geospatial-temporal raster data with the - |same flexibility and ease as any other Spark Catalyst data type. At its core - |is a user-defined type (UDT) called TileUDT, which encodes a GeoTrellis Tile - |in a form the Spark Catalyst engine can process. Furthermore, we extend the - |definition of a DataFrame to encompass some additional invariants, allowing - |for geospatial operations within and between RasterFrames to occur, while - |still maintaining necessary geo-referencing constructs. - """.stripMargin - -spPublishLocal := { - // This unfortunate override is necessary because - // the ivy resolver in pyspark defaults to the cache more - // frequently than we'd like. - val id = (projectID in spPublishLocal).value - val home = ivyPaths.value.ivyHome - .getOrElse(io.Path.userHome / ".ivy2") - val cacheDir = home / "cache" / id.organization / id.name - IO.delete(cacheDir) - spPublishLocal.value -} - -//-------------------------------------------------------------------- -// Python Build -//-------------------------------------------------------------------- - -lazy val pythonSource = settingKey[File]("Default Python source directory.") -pythonSource := baseDirectory.value / "python" - -def sparkFiles(dir: File):Seq[File] = Seq("metastore_db", - "spark-warehouse", - "derby.log").map(f => dir / f) - -cleanFiles ++= - sparkFiles(baseDirectory.value) ++ - sparkFiles(pythonSource.value) ++ - Seq( - ".eggs", - ".pytest_cache", - "build", - "dist", - "pyrasterframes.egg-info" - ).map(f => pythonSource.value / f) - -val Python = config("python") - -Python / target := target.value / "python-dist" - -lazy val pyZip = taskKey[File]("Build pyrasterframes zip.") - -// Alias -pyZip := (Python / packageBin).value - -Python / packageBin / artifact := { - val java = (Compile / packageBin / artifact).value - java.withType("zip").withClassifier(Some("python")).withExtension("zip") -} - -Python / packageBin / artifactPath := { - val dir = (Python / target).value - val art = (Python / packageBin / artifact).value - val ver = version.value - dir / s"${art.name}-python-$ver.zip" -} - -addArtifact(Python / packageBin / artifact, Python / packageBin) - -val pysparkCmd = taskKey[Unit]("Builds pyspark package and emits command string for running pyspark with package") - -lazy val pyTest = taskKey[Int]("Run pyrasterframes tests. Return result code.") - -lazy val pyExamples = taskKey[Unit]("Run pyrasterframes examples.") - -lazy val pyWheel = taskKey[Unit]("Creates a Python .whl file") - -lazy val spJarFile = Def.taskDyn { - if (spShade.value) { - Def.task((assembly in spPackage).value) - } else { - Def.task(spPackage.value) - } -} - -def gatherFromDir(root: sbt.File, dirName: String) = { - val pyDir = root / dirName - IO.listFiles(pyDir, GlobFilter("*.py") | GlobFilter("*.rst")) - .map(f => (f, s"$dirName/${f.getName}")) -} - -Python / packageBin := { - val jar = (`package` in Compile).value - val root = pythonSource.value - val license = root / "LICENSE.md" - val files = gatherFromDir(root, "pyrasterframes") ++ - gatherFromDir(root, "geomesa_pyspark") ++ - Seq(jar, license).map(f => (f, "pyrasterframes/" + f.getName)) - val zipFile = (Python / packageBin / artifactPath).value - IO.zip(files, zipFile) - zipFile -} +val nbInclude = Def.setting[FileFilter](GlobFilter("*.ipynb")) -pysparkCmd := { - val _ = spPublishLocal.value - val id = (projectID in spPublishLocal).value - val args = "pyspark" :: "--packages" :: s"${id.organization}:${id.name}:${id.revision}" :: Nil - streams.value.log.info("PySpark Command:\n" + args.mkString(" ")) - // --conf spark.jars.ivy=(ivyPaths in pysparkCmd).value.... +lazy val pyNotebooks = taskKey[Seq[File]]("Convert relevant scripts into notebooks") +pyNotebooks := { + val _ = pySetup.toTask(" notebooks").value + ((Python / doc / target).value ** "*.ipynb").get() } -ivyPaths in pysparkCmd := ivyPaths.value.withIvyHome(target.value / "ivy") - -pyTest := { - val _ = assembly.value +lazy val pySparkCmd = taskKey[Unit]("Create build and emit command to run in pyspark") +pySparkCmd := { val s = streams.value - s.log.info("Running python tests...") - val wd = pythonSource.value - Process("python setup.py test", wd).! + val jvm = assembly.value + val py = (Python / packageBin).value + val script = IO.createTemporaryDirectory / "pyrf_init.py" + IO.write(script, """ +import pyrasterframes +from pyrasterframes.rasterfunctions import * +""") + val msg = s"PYTHONSTARTUP=$script pyspark --jars $jvm --py-files $py" + s.log.debug(msg) + println(msg) } -Test / executeTests := { - val standard = (Test / executeTests).value - standard.overall match { - case TestResult.Passed ⇒ - val resultCode = pyTest.value - val msg = resultCode match { - case 1 ⇒ "There are Python test failures." - case 2 ⇒ "Python test execution was interrupted." - case 3 ⇒ "Internal error during Python test execution." - case 4 ⇒ "PyTest usage error." - case 5 ⇒ "No Python tests found." - case x if (x != 0) ⇒ "Unknown error while running Python tests." - case _ ⇒ "PyRasterFrames tests successfully completed." - } - val pySummary = Summary("pyrasterframes", msg) - val summaries = standard.summaries ++ Iterable(pySummary) - // Would be cool to derive this from the python output... - val result = if(resultCode == 0) { - new SuiteResult( - TestResult.Passed, - passedCount = 1, - failureCount = 0, - errorCount = 0, - skippedCount = 0, - ignoredCount = 0, - canceledCount = 0, - pendingCount = 0 - ) - } - else { - new SuiteResult( - TestResult.Failed, - passedCount = 0, - failureCount = 1, - errorCount = 0, - skippedCount = 0, - ignoredCount = 0, - canceledCount = 0, - pendingCount = 0 - ) - } - standard.copy(overall = result.result, summaries = summaries, events = standard.events + ("PyRasterFramesTests" -> result)) - case _ ⇒ - val pySummary = Summary("pyrasterframes", "tests skipped due to scalatest failures") - standard.copy(summaries = standard.summaries ++ Iterable(pySummary)) - } -} -pyWheel := { - val s = streams.value - val wd = pythonSource.value - Process("python setup.py bdist_wheel", wd) ! s.log - val whl = IO.listFiles(pythonSource.value / "dist", GlobFilter("*.whl"))(0) - IO.move(whl, (Python / target).value / whl.getName) -} -pyExamples := { - val _ = spPublishLocal.value - val s = streams.value - val wd = pythonSource.value - Process("python setup.py examples", wd) ! s.log -} diff --git a/pyrasterframes/python/README.rst b/pyrasterframes/python/README.rst deleted file mode 100644 index 8c80fb496..000000000 --- a/pyrasterframes/python/README.rst +++ /dev/null @@ -1,75 +0,0 @@ -============== -PyRasterFrames -============== - -PyRasterFrames provides a Python API for RasterFrames! - ------------- -SBT Commands ------------- - -Build the shaded JAR: - - ``$ sbt pyrasterframes/spPublishLocal`` - -Run tests: - - ``$ sbt pyrasterframes/pyTest`` - -Run examples: - - ``$ sbt pyrasterframes/pyExamples`` - - ---------------- -Python Commands ---------------- - -Install the python package (for development / local use): - - ``$ pip install -e python`` - - -To run tests and examples, ``$ cd python``, then: - -Run tests (if no Scala code has changed): - - ``$ python setup.py test`` - - To run verbosely: - ``$ python setup.py test --addopts -s`` - - To run a specific test: - ``$ python setup.py test --addopts "-k my_test_name"`` - - -Run examples: - - ``$ python setup.py examples [--e EXAMPLENAME,EXAMPLENAME]`` - - ------------ -Spark Usage ------------ - -Get a Spark REPL (after running ``$ sbt pyrasterframes/spPublishLocal``): - - ``$ pyspark --packages io.astraea:pyrasterframes:$VERSION --master local[2]`` - -You can then try some of the commands in ``python/tests/PyRasterFramesTests.py``. - -Submit a script (from the ``python`` directory): - - ``$ spark-submit --packages io.astraea:pyrasterframes:$VERSION --master local[2] \`` - ``examples/CreatingRasterFrames.py`` - -To initialize PyRasterFrames: - - >>> from pyrasterframes import * - >>> spark = SparkSession.builder \ - ... .master("local[*]") \ - ... .appName("Using RasterFrames") \ - ... .config("spark.some.config.option", "some-value") \ - ... .getOrCreate() \ - ... .withRasterFrames() - diff --git a/pyrasterframes/python/examples/Clustering.py b/pyrasterframes/python/examples/Clustering.py deleted file mode 100644 index 8566d6276..000000000 --- a/pyrasterframes/python/examples/Clustering.py +++ /dev/null @@ -1,83 +0,0 @@ - -#examples_setup -from . import example_session, resource_dir -#examples_setup - -#py_cl_imports -from pyrasterframes import * -from pyrasterframes.rasterfunctions import * -from pyspark.sql import * -from pyspark.ml.feature import VectorAssembler -from pyspark.ml.clustering import KMeans -from pyspark.ml import Pipeline -#py_cl_imports - -#py_cl_create_session -spark = example_session().withRasterFrames() -#py_cl_create_session - -# The first step is to load multiple bands of imagery and construct -# a single RasterFrame from them. -filenamePattern = "L8-B{}-Elkton-VA.tiff" -bandNumbers = range(1, 8) -bandColNames = list(map(lambda n: 'band_{}'.format(n), bandNumbers)) - -# For each identified band, load the associated image file -from functools import reduce -joinedRF = reduce(lambda rf1, rf2: rf1.asRF().spatialJoin(rf2.drop('bounds').drop('metadata').drop('extent').drop('crs')), - map(lambda bf: spark.read.geotiff(bf[1]) \ - .withColumnRenamed('tile', 'band_{}'.format(bf[0])), - map(lambda b: (b, resource_dir.joinpath(filenamePattern.format(b)).as_uri()), bandNumbers))) - -# We should see a single spatial_key column along with columns of tiles. -joinedRF.printSchema() - - -# SparkML requires that each observation be in its own row, and those -# observations be packed into a single `Vector`. The first step is to -# "explode" the tiles into a single row per cell/pixel -exploder = TileExploder() - - -# To "vectorize" the the band columns we use the SparkML `VectorAssembler` -assembler = VectorAssembler() \ - .setInputCols(bandColNames) \ - .setOutputCol("features") - -# Configure our clustering algorithm -k = 5 -kmeans = KMeans().setK(k) - -# Combine the two stages -pipeline = Pipeline().setStages([exploder, assembler, kmeans]) - - -# Compute clusters -model = pipeline.fit(joinedRF) - -# Run the data through the model to assign cluster IDs to each -clustered = model.transform(joinedRF) -clustered.show(8) - - -# If we want to inspect the model statistics, the SparkML API requires us to go -# through this unfortunate contortion: -clusterResults = list(filter(lambda x: str(x).startswith('KMeans'), model.stages))[0] - -# Compute sum of squared distances of points to their nearest center -metric = clusterResults.computeCost(clustered) -print("Within set sum of squared errors: %s" % metric) - -tlm = joinedRF.tileLayerMetadata() -layout = tlm['layoutDefinition']['tileLayout'] - -retiled = clustered.groupBy('spatial_key').agg( - assemble_tile('column_index', 'row_index', 'prediction', - layout['tileCols'], layout['tileRows'], 'int8') -) - -rf = retiled.asRF('spatial_key', tlm) -rf.printSchema() -rf.show() - -spark.stop() \ No newline at end of file diff --git a/pyrasterframes/python/examples/CreatingRasterFrames.py b/pyrasterframes/python/examples/CreatingRasterFrames.py deleted file mode 100644 index 429bb0f48..000000000 --- a/pyrasterframes/python/examples/CreatingRasterFrames.py +++ /dev/null @@ -1,32 +0,0 @@ - -#examples_setup -from . import resource_dir, example_session -#examples_setup - -#py_crf_imports -from pyrasterframes import * -from pyspark.sql import * -#py_crf_imports - -#py_crf_create_session -spark = example_session().withRasterFrames() -#py_crf_create_session - -#py_crf_more_imports -# No additional imports needed for Python. -#py_crf_more_imports - -#py_crf_create_rasterframe -rf = spark.read.geotiff(resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri()) -rf.show(5, False) -#py_crf_create_rasterframe - -#py_crf_metadata -rf.tileColumns() - -rf.spatialKeyColumn() - -rf.temporalKeyColumn() - -rf.tileLayerMetadata() -#py_crf_metadata diff --git a/pyrasterframes/python/examples/MeanValue.py b/pyrasterframes/python/examples/MeanValue.py deleted file mode 100644 index 2c7308c43..000000000 --- a/pyrasterframes/python/examples/MeanValue.py +++ /dev/null @@ -1,25 +0,0 @@ - -#examples_setup -from . import resource_dir, example_session -#examples_setup - - -#py_mv_imports -from pyrasterframes import * -from pyrasterframes.rasterfunctions import * -from pyspark.sql import * -#py_mv_imports - -#py_mv_create_session -spark = example_session().withRasterFrames() -#py_mv_create_session - -#py_mv_create_rasterframe -rf = spark.read.geotiff(resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri()) -rf.show(5, False) -#py_mv_create_rasterframe - -#py_mv_find_mean -tileCol = 'tile' -rf.agg(agg_no_data_cells(tileCol), agg_data_cells(tileCol), agg_mean(tileCol)).show(5, False) -#py_mv_find_mean \ No newline at end of file diff --git a/pyrasterframes/python/examples/NDVI.py b/pyrasterframes/python/examples/NDVI.py deleted file mode 100644 index f98619d27..000000000 --- a/pyrasterframes/python/examples/NDVI.py +++ /dev/null @@ -1,30 +0,0 @@ - - -#examples_setup -from . import resource_dir, example_session -#examples_setup - -#py_nvdi_imports -from pyrasterframes import * -from pyrasterframes.rasterfunctions import * -from pyspark.sql import * -from pyspark.sql.functions import udf -#py_nvdi_imports - -#py_nvdi_create_session -spark = example_session().withRasterFrames() -#py_nvdi_create_session - -#py_ndvi_create_rasterframe -redBand = spark.read.geotiff(resource_dir.joinpath('L8-B4-Elkton-VA.tiff').as_uri()).withColumnRenamed('tile', 'red_band') -nirBand = spark.read.geotiff(resource_dir.joinpath('L8-B5-Elkton-VA.tiff').as_uri()).withColumnRenamed('tile', 'nir_band') -#py_ndvi_create_rasterframe - -#py_ndvi_column -rf = redBand.asRF().spatialJoin(nirBand.asRF()) \ - .withColumn("ndvi", normalized_difference('red_band', 'nir_band')) -rf.printSchema() -rf.show(20) -#py_ndvi_column - -spark.stop() \ No newline at end of file diff --git a/pyrasterframes/python/examples/__init__.py b/pyrasterframes/python/examples/__init__.py deleted file mode 100644 index b084e150d..000000000 --- a/pyrasterframes/python/examples/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -#examples_setup -from pathlib import Path -from pyspark.sql import SparkSession - -__all__ = ["resource_dir", "example_session"] - -jarpath = list(Path('../target').resolve().glob('**/pyrasterframes*.jar')) -if len(jarpath) > 0: - pyJar = jarpath[0].as_uri() - def example_session(): - return (SparkSession.builder - .master("local[*]") - .appName("RasterFrames") - .config('spark.driver.extraClassPath', pyJar) - .config('spark.executor.extraClassPath', pyJar) - .config("spark.ui.enabled", "false") - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - .config("spark.kryo.registrator", "astraea.spark.rasterframes.util.RFKryoRegistrator") - .config("spark.kryoserializer.buffer.max", "500m") - .getOrCreate()) - - -# hard-coded relative path for resources -resource_dir = Path('./static').resolve() -#examples_setup diff --git a/pyrasterframes/python/geomesa_pyspark/types.py b/pyrasterframes/python/geomesa_pyspark/types.py deleted file mode 100644 index f70f010c7..000000000 --- a/pyrasterframes/python/geomesa_pyspark/types.py +++ /dev/null @@ -1,68 +0,0 @@ -"""*********************************************************************** - This file was created by Astraea, Inc., 2018 from an excerpt of the - original: - - Copyright (c) 2013-2018 Commonwealth Computer Research, Inc. - All rights reserved. This program and the accompanying materials - are made available under the terms of the Apache License, Version 2.0 - which accompanies this distribution and is available at - http://www.opensource.org/licenses/apache2.0.php. -+ ***********************************************************************/""" - -from pyspark.sql.types import UserDefinedType -from pyspark.sql import Row -from pyspark.sql.types import * -from pyrasterframes.context import RFContext - -class GeometryUDT(UserDefinedType): - @classmethod - def sqlType(self): - # return StructField("wkb", BinaryType(), False) - return StructType([StructField("wkb", BinaryType(), True)]) - - @classmethod - def module(cls): - return 'geomesa_pyspark.types' - - @classmethod - def scalaUDT(cls): - return 'org.apache.spark.sql.jts.' + cls.__name__ - - def serialize(self, obj): - if (obj is None): return None - return Row(obj.toBytes) - - def deserialize(self, datum): - return RFContext._jvm_mirror().generate_geometry(datum[0]) - - -class PointUDT(GeometryUDT): - pass - - -class LineStringUDT(GeometryUDT): - pass - - -class PolygonUDT(GeometryUDT): - pass - - -class MultiPointUDT(GeometryUDT): - pass - - -class MultiLineStringUDT(GeometryUDT): - pass - - -class MultiPolygonUDT(GeometryUDT): - pass - - -class GeometryUDT(GeometryUDT): - pass - - -class GeometryCollectionUDT(GeometryUDT): - pass diff --git a/pyrasterframes/python/pyrasterframes/README.md b/pyrasterframes/python/pyrasterframes/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/pyrasterframes/python/pyrasterframes/README.rst b/pyrasterframes/python/pyrasterframes/README.rst deleted file mode 100644 index cf9330217..000000000 --- a/pyrasterframes/python/pyrasterframes/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -PyRasterFrames --------------- - -PyRasterFrames provides a Python API for RasterFrames! - -To initialize PyRasterFrames: - - >>> from pyrasterframes import * - >>> spark = SparkSession.builder \ - ... .master("local[*]") \ - ... .appName("Using RasterFrames") \ - ... .config("spark.some.config.option", "some-value") \ - ... .getOrCreate() \ - ... .withRasterFrames() - diff --git a/pyrasterframes/python/pyrasterframes/__init__.py b/pyrasterframes/python/pyrasterframes/__init__.py deleted file mode 100644 index b886bfabd..000000000 --- a/pyrasterframes/python/pyrasterframes/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Module initialization for PyRasterFrames. This is where much of the cool stuff is -appended to PySpark classes. -""" - -from __future__ import absolute_import -from pyspark import SparkContext -from pyspark.sql import SparkSession, DataFrame, DataFrameReader -from pyspark.sql.column import _to_java_column - -# Import RasterFrame types and functions -from .types import * -from . import rasterfunctions -from .context import RFContext - -__all__ = ['RasterFrame', 'TileExploder'] - - -def _rf_init(spark_session): - """ Adds RasterFrames functionality to PySpark session.""" - if not hasattr(spark_session, "rasterframes"): - spark_session.rasterframes = RFContext(spark_session) - spark_session.sparkContext._rf_context = spark_session.rasterframes - - return spark_session - - -def _reader(df_reader, format_key, path, **options): - """ Loads the file of the given type at the given path.""" - df = df_reader.format(format_key).load(path, **options) - return _convertDF(df) - - -def _convertDF(df, sp_key = None, metadata = None): - ctx = SparkContext._active_spark_context._rf_context - - if sp_key is None: - return RasterFrame(ctx._jrfctx.asRF(df._jdf), ctx._spark_session) - else: - import json - return RasterFrame(ctx._jrfctx.asRF( - df._jdf, _to_java_column(sp_key), json.dumps(metadata)), ctx._spark_session) - - -# Patch new method on SparkSession to mirror Scala approach -SparkSession.withRasterFrames = _rf_init - -# Add the 'asRF' method to pyspark DataFrame -DataFrame.asRF = _convertDF - -# Add DataSource convenience methods to the DataFrameReader -# TODO: make sure this supports **options -DataFrameReader.geotiff = lambda df_reader, path: _reader(df_reader, "geotiff", path) -DataFrameReader.geotrellis = lambda df_reader, path: _reader(df_reader, "geotrellis", path) - - - diff --git a/pyrasterframes/python/pyrasterframes/context.py b/pyrasterframes/python/pyrasterframes/context.py deleted file mode 100644 index 9f238c225..000000000 --- a/pyrasterframes/python/pyrasterframes/context.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module contains access to the jvm SparkContext with RasterFrame support. -""" - -from pyspark import SparkContext - -__all__ = ['RFContext'] - - -class RFContext(object): - """ - Entrypoint to RasterFrames services - """ - def __init__(self, spark_session): - self._spark_session = spark_session - self._gateway = spark_session.sparkContext._gateway - self._jvm = self._gateway.jvm - jsess = self._spark_session._jsparkSession - self._jrfctx = self._jvm.astraea.spark.rasterframes.py.PyRFContext(jsess) - - def list_to_seq(self, py_list): - conv = self.lookup('listToSeq') - return conv(py_list) - - def lookup(self, function_name): - return getattr(self._jrfctx, function_name) - - @staticmethod - def active(): - """ - Get the active Pythono RFContext and throw an error if it is not enabled for RasterFrames. - """ - sc = SparkContext._active_spark_context - if not hasattr(sc, '_rf_context'): - raise AttributeError( - "RasterFrames have not been enabled for the active session. Call 'SparkSession.withRasterFrames()'.") - return sc._rf_context - - @staticmethod - def _jvm_mirror(): - """ - Get the active Scala PyRFContext and throw an error if it is not enabled for RasterFrames. - """ - return RFContext.active()._jrfctx - diff --git a/pyrasterframes/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/python/pyrasterframes/rasterfunctions.py deleted file mode 100644 index 589b256d8..000000000 --- a/pyrasterframes/python/pyrasterframes/rasterfunctions.py +++ /dev/null @@ -1,368 +0,0 @@ -""" -This module creates explicit Python functions that map back to the existing Scala -implementations. Most functions are standard Column functions, but those with unique -signatures are handled here as well. -""" - - -from __future__ import absolute_import -from pyspark.sql.types import * -from pyspark.sql.column import Column, _to_java_column -from .context import RFContext - - -THIS_MODULE = 'pyrasterframes' - - -def _context_call(name, *args): - f = RFContext.active().lookup(name) - return f(*args) - - -def _celltype(cellTypeStr): - """ Convert the string cell type to the expected CellType object.""" - return _context_call('cell_type', cellTypeStr) - - -def _create_assembleTile(): - """ Create a function mapping to the Scala implementation.""" - def _(colIndex, rowIndex, cellData, numCols, numRows, cellType): - jfcn = RFContext.active().lookup('assemble_tile') - return Column(jfcn(_to_java_column(colIndex), _to_java_column(rowIndex), _to_java_column(cellData), numCols, numRows, _celltype(cellType))) - _.__name__ = 'assemble_tile' - _.__doc__ = "Create a Tile from a column of cell data with location indices" - _.__module__ = THIS_MODULE - return _ - - -def _create_arrayToTile(): - """ Create a function mapping to the Scala implementation.""" - def _(arrayCol, numCols, numRows): - jfcn = RFContext.active().lookup('array_to_tile') - return Column(jfcn(_to_java_column(arrayCol), numCols, numRows)) - _.__name__ = 'array_to_tile' - _.__doc__ = "Convert array in `arrayCol` into a Tile of dimensions `numCols` and `numRows'" - _.__module__ = THIS_MODULE - return _ - - -def _create_convertCellType(): - """ Create a function mapping to the Scala implementation.""" - def _(tileCol, cellType): - jfcn = RFContext.active().lookup('convert_cell_type') - return Column(jfcn(_to_java_column(tileCol), _celltype(cellType))) - _.__name__ = 'convert_cell_type' - _.__doc__ = "Convert the numeric type of the Tiles in `tileCol`" - _.__module__ = THIS_MODULE - return _ - - -def _create_makeConstantTile(): - """ Create a function mapping to the Scala implementation.""" - def _(value, cols, rows, cellType): - jfcn = RFContext.active().lookup('make_constant_tile') - return Column(jfcn(value, cols, rows, cellType)) - _.__name__ = 'make_constant_tile' - _.__doc__ = "Constructor for constant tile column" - _.__module__ = THIS_MODULE - return _ - - -def _create_tileZeros(): - """ Create a function mapping to the Scala implementation.""" - def _(cols, rows, cellType = 'float64'): - jfcn = RFContext.active().lookup('tile_zeros') - return Column(jfcn(cols, rows, cellType)) - _.__name__ = 'tile_zeros' - _.__doc__ = "Create column of constant tiles of zero" - _.__module__ = THIS_MODULE - return _ - - -def _create_tileOnes(): - """ Create a function mapping to the Scala implementation.""" - def _(cols, rows, cellType = 'float64'): - jfcn = RFContext.active().lookup('tile_ones') - return Column(jfcn(cols, rows, cellType)) - _.__name__ = 'tile_ones' - _.__doc__ = "Create column of constant tiles of one" - _.__module__ = THIS_MODULE - return _ - - -def _create_rasterize(): - """ Create a function mapping to the Scala rasterize function. """ - def _(geometryCol, boundsCol, valueCol, numCols, numRows): - jfcn = RFContext.active().lookup('rasterize') - return Column(jfcn(_to_java_column(geometryCol), _to_java_column(boundsCol), _to_java_column(valueCol), numCols, numRows)) - _.__name__ = 'rasterize' - _.__doc__ = 'Create a tile where cells in the grid defined by cols, rows, and bounds are filled with the given value.' - _.__module__ = THIS_MODULE - return _ - - -def _create_reproject_geometry(): - """ Create a function mapping to the Scala reproject_geometry function. """ - def _(geometryCol, srcCRSName, dstCRSName): - jfcn = RFContext.active().lookup('reproject_geometry') - return Column(jfcn(_to_java_column(geometryCol), srcCRSName, dstCRSName)) - _.__name__ = 'reproject_geometry' - _.__doc__ = """Reproject a column of geometry given the CRS names of the source and destination. -Currently supported registries are EPSG, ESRI, WORLD, NAD83, & NAD27. -An example of a valid CRS name is EPSG:3005. -""" - _.__module__ = THIS_MODULE - return _ - - -def _create_explode_tiles(): - """ Create a function mapping to Scala explode_tiles function """ - def _(*args): - jfcn = RFContext.active().lookup('explode_tiles') - jcols = [_to_java_column(arg) for arg in args] - return Column(jfcn(RFContext.active().list_to_seq(jcols))) - _.__name__ = 'explode_tiles' - _.__doc__ = 'Create a row for each cell in Tile.' - _.__module__ = THIS_MODULE - return _ - - -def _create_explode_tiles_sample(): - """ Create a function mapping to Scala explode_tiles_sample function""" - def _(sample_frac, seed, *tile_cols): - jfcn = RFContext.active().lookup('explode_tiles_sample') - jcols = [_to_java_column(arg) for arg in tile_cols] - return Column(jfcn(sample_frac, seed, RFContext.active().list_to_seq(jcols))) - - _.__name__ = 'explode_tiles_sample' - _.__doc__ = 'Create a row for a sample of cells in Tile columns.' - _.__module__ = THIS_MODULE - return _ - - -def _create_maskByValue(): - """ Create a function mapping to Scala mask_by_value function """ - def _(data_tile, mask_tile, mask_value): - jfcn = RFContext.active().lookup('mask_by_value') - return Column(jfcn(_to_java_column(data_tile), _to_java_column(mask_tile), _to_java_column(mask_value))) - _.__name__ = 'mask_by_value' - _.__doc__ = 'Generate a tile with the values from the data tile, but where cells in the masking tile contain the masking value, replace the data value with NODATA.' - _.__module__ = THIS_MODULE - return _ - - -_rf_unique_functions = { - 'array_to_tile': _create_arrayToTile(), - 'assemble_tile': _create_assembleTile(), - 'cell_types': lambda: _context_call('cell_types'), - 'convert_cell_type': _create_convertCellType(), - 'explode_tiles': _create_explode_tiles(), - 'explode_tiles_sample': _create_explode_tiles_sample(), - 'make_constant_tile': _create_makeConstantTile(), - 'mask_by_value': _create_maskByValue(), - 'rasterize': _create_rasterize(), - 'reproject_geometry': _create_reproject_geometry(), - 'tile_ones': _create_tileOnes(), - 'tile_zeros': _create_tileZeros(), -} - - -_rf_column_scalar_functions = { - 'with_no_data': 'Assign a `NoData` value to the Tiles in the given Column.', - 'local_add_scalar': 'Add a scalar to a Tile', - 'local_add_scalar_int': 'Add a scalar to a Tile', - 'local_subtract_scalar': 'Subtract a scalar from a Tile', - 'local_subtract_scalar_int': 'Subtract a scalar from a Tile', - 'local_multiply_scalar': 'Multiply a Tile by a scalar', - 'local_multiply_scalar_int': 'Multiply a Tile by a scalar', - 'local_divide_scalar': 'Divide a Tile by a scalar', - 'local_divide_scalar_int': 'Divide a Tile by a scalar', - 'local_less_scalar': 'Return a Tile with values equal 1 if the cell is less than a scalar, otherwise 0', - 'local_less_scalar_int': 'Return a Tile with values equal 1 if the cell is less than a scalar, otherwise 0', - 'local_less_equal_scalar': 'Return a Tile with values equal 1 if the cell is less than or equal to a scalar, otherwise 0', - 'local_less_equal_scalar_int': 'Return a Tile with values equal 1 if the cell is less than or equal to a scalar, otherwise 0', - 'local_greater_scalar': 'Return a Tile with values equal 1 if the cell is greater than a scalar, otherwise 0', - 'local_greater_scalar_int': 'Return a Tile with values equal 1 if the cell is greater than a scalar, otherwise 0', - 'local_greater_equal_scalar': 'Return a Tile with values equal 1 if the cell is greater than or equal to a scalar, otherwise 0', - 'local_greater_equal_scalar_int': 'Return a Tile with values equal 1 if the cell is greater than or equal to a scalar, otherwise 0', - 'local_equal_scalar': 'Return a Tile with values equal 1 if the cell is equal to a scalar, otherwise 0', - 'local_equal_scalar_int': 'Return a Tile with values equal 1 if the cell is equal to a scalar, otherwise 0', - 'local_unequal_scalar': 'Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0', - 'local_unequal_scalar_int': 'Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0', -} - - -_rf_column_functions = { - # ------- RasterFrames functions ------- - 'tile_dimensions': 'Query the number of (cols, rows) in a Tile.', - 'envelope': 'Extracts the bounding box (envelope) of the geometry.', - 'tile_to_int_array': 'Flattens Tile into an array of integers. Deprecated in favor of `tile_to_array_int`.', - 'tile_to_double_array': 'Flattens Tile into an array of doubles. Deprecated in favor of `tile_to_array_double`', - 'tile_to_array_int': 'Flattens Tile into an array of integers.', - 'tile_to_array_double': 'Flattens Tile into an array of doubles.', - 'cell_type': 'Extract the Tile\'s cell type', - 'is_no_data_tile': 'Report if the Tile is entirely NODDATA cells', - 'agg_approx_histogram': 'Compute the full column aggregate floating point histogram', - 'agg_stats': 'Compute the full column aggregate floating point statistics', - 'agg_mean': 'Computes the column aggregate mean', - 'agg_data_cells': 'Computes the number of non-NoData cells in a column', - 'agg_no_data_cells': 'Computes the number of NoData cells in a column', - 'tile_histogram': 'Compute the Tile-wise histogram', - 'tile_mean': 'Compute the Tile-wise mean', - 'tile_sum': 'Compute the Tile-wise sum', - 'tile_min': 'Compute the Tile-wise minimum', - 'tile_max': 'Compute the Tile-wise maximum', - 'tile_stats': 'Compute the Tile-wise floating point statistics', - 'render_ascii': 'Render ASCII art of tile', - 'no_data_cells': 'Count of NODATA cells', - 'data_cells': 'Count of cells with valid data', - 'local_add': 'Add two Tiles', - 'local_subtract': 'Subtract two Tiles', - 'local_multiply': 'Multiply two Tiles', - 'local_divide': 'Divide two Tiles', - 'normalized_difference': 'Compute the normalized difference of two tiles', - 'local_agg_stats': 'Compute cell-local aggregate descriptive statistics for a column of Tiles.', - 'agg_local_max': 'Compute the cell-wise/local max operation between Tiles in a column.', - 'agg_local_min': 'Compute the cellwise/local min operation between Tiles in a column.', - 'agg_local_mean': 'Compute the cellwise/local mean operation between Tiles in a column.', - 'agg_local_data_cells': 'Compute the cellwise/local count of non-NoData cells for all Tiles in a column.', - 'agg_local_no_data_cells': 'Compute the cellwise/local count of NoData cells for all Tiles in a column.', - 'mask': 'Where the mask (second) tile contains NODATA, replace values in the source (first) tile with NODATA.', - 'inverse_mask': 'Where the mask (second) tile DOES NOT contain NODATA, replace values in the source (first) tile with NODATA.', - 'local_less': 'Cellwise less than comparison between two tiles', - 'local_less_equal': 'Cellwise less than or equal to comparison between two tiles', - 'local_greater': 'Cellwise greater than comparison between two tiles', - 'local_greater_equal': 'Cellwise greater than or equal to comparison between two tiles', - 'local_equal': 'Cellwise equality comparison between two tiles', - 'local_unequal': 'Cellwise inequality comparison between two tiles', - 'round': 'Round cell values to the nearest integer without changing the cell type', - 'log': 'Performs cell-wise natural logarithm', - 'log10': 'Performs cell-wise logartithm with base 10', - 'log2': 'Performs cell-wise logartithm with base 2', - 'log1p': 'Performs natural logarithm of cell values plus one', - 'exp': 'Performs cell-wise exponential', - 'exp2': 'Compute 2 to the power of cell values', - 'exp10': 'Compute 10 to the power of cell values', - 'expm1': 'Performs cell-wise exponential, then subtract one', - 'resample': 'Resample tile to different size based on scalar factor or tile whose dimension to match', - - # ------- JTS functions ------- - # spatial constructors - 'st_geomFromGeoHash': '', - 'st_geomFromWKT': '', - 'st_geomFromWKB': '', - 'st_lineFromText': '', - 'st_makeBox2D': '', - 'st_makeBBox': '', - 'st_makePolygon': '', - 'st_makePoint': '', - 'st_makeLine': '', - 'st_makePointM': '', - 'st_mLineFromText': '', - 'st_mPointFromText': '', - 'st_mPolyFromText': '', - 'st_point': '', - 'st_pointFromGeoHash': '', - 'st_pointFromText': '', - 'st_pointFromWKB': '', - 'st_polygon': '', - 'st_polygonFromText': '', - # spatial converters - 'st_castToPoint': '', - 'st_castToPolygon': '', - 'st_castToLineString': '', - 'st_byteArray': '', - # spatial accessors - 'st_boundary': '', - 'st_coordDim': '', - 'st_dimension': '', - 'st_envelope': '', - 'st_exteriorRing': '', - 'st_geometryN': '', - 'st_geometryType': '', - 'st_interiorRingN': '', - 'st_isClosed': '', - 'st_isCollection': '', - 'st_isEmpty': '', - 'st_isRing': '', - 'st_isSimple': '', - 'st_isValid': '', - 'st_numGeometries': '', - 'st_numPoints': '', - 'st_pointN': '', - 'st_x': '', - 'st_y': '', - # spatial outputs - 'st_asBinary': '', - 'st_asGeoJSON': '', - 'st_asLatLonText': '', - 'st_asText': '', - 'st_geoHash': '', - # spatial processors - 'st_bufferPoint': '', - 'st_antimeridianSafeGeom': '', - # spatial relations - 'st_translate': '', - 'st_contains': '', - 'st_covers': '', - 'st_crosses': '', - 'st_disjoint': '', - 'st_equals': '', - 'st_intersects': '', - 'st_overlaps': '', - 'st_touches': '', - 'st_within': '', - 'st_relate': '', - 'st_relateBool': '', - 'st_area': '', - 'st_closestPoint': '', - 'st_centroid': '', - 'st_distance': '', - 'st_distanceSphere': '', - 'st_length': '', - 'st_aggregateDistanceSphere': '', - 'st_lengthSphere': '', -} - - -__all__ = list(_rf_column_functions.keys()) + \ - list(_rf_column_scalar_functions.keys()) + \ - list(_rf_unique_functions.keys()) - - -def _create_column_function(name, doc=""): - """ Create a mapping to Scala UDF for a column function by name""" - def _(*args): - jfcn = RFContext.active().lookup(name) - jcols = [_to_java_column(arg) for arg in args] - return Column(jfcn(*jcols)) - _.__name__ = name - _.__doc__ = doc - _.__module__ = THIS_MODULE - return _ - - -def _create_columnScalarFunction(name, doc=""): - """ Create a mapping to Scala UDF for a (column, scalar) -> column function by name""" - def _(col, scalar): - jfcn = RFContext.active().lookup(name) - return Column(jfcn(_to_java_column(col), scalar)) - _.__name__ = name - _.__doc__ = doc - _.__module__ = THIS_MODULE - return _ - - -def _register_functions(): - """ Register each function in the scope""" - for name, doc in _rf_column_functions.items(): - globals()[name] = _create_column_function(name, doc) - - for name, doc in _rf_column_scalar_functions.items(): - globals()[name] = _create_columnScalarFunction(name, doc) - - for name, func in _rf_unique_functions.items(): - globals()[name] = func - - -_register_functions() \ No newline at end of file diff --git a/pyrasterframes/python/pyrasterframes/types.py b/pyrasterframes/python/pyrasterframes/types.py deleted file mode 100644 index b82cfb70c..000000000 --- a/pyrasterframes/python/pyrasterframes/types.py +++ /dev/null @@ -1,190 +0,0 @@ -""" -This module contains all types relevant to PyRasterFrames. Classes in this module are -meant to provide smoother pathways between the jvm and Python, and whenever possible, -the implementations take advantage of the existing Scala functionality. The RasterFrame -class here provides the PyRasterFrames entry point. -""" - -from pyspark.sql.types import UserDefinedType -from pyspark import SparkContext -from pyspark.sql import SparkSession, DataFrame, Column, Row -from pyspark.sql.types import * -from pyspark.ml.wrapper import JavaTransformer -from pyspark.ml.util import JavaMLReadable, JavaMLWritable -from .context import RFContext - -__all__ = ['RasterFrame', 'TileUDT', 'RasterSourceUDT', 'TileExploder', 'NoDataFilter'] - - -class RasterFrame(DataFrame): - def __init__(self, jdf, spark_session): - DataFrame.__init__(self, jdf, spark_session) - self._jrfctx = spark_session.rasterframes._jrfctx - - def tileColumns(self): - """ - Fetches columns of type Tile. - :return: One or more Column instances associated with Tiles. - """ - cols = self._jrfctx.tileColumns(self._jdf) - return [Column(c) for c in cols] - - def spatialKeyColumn(self): - """ - Fetch the tagged spatial key column. - :return: Spatial key column - """ - col = self._jrfctx.spatialKeyColumn(self._jdf) - return Column(col) - - def temporalKeyColumn(self): - """ - Fetch the temporal key column, if any. - :return: Temporal key column, or None. - """ - col = self._jrfctx.temporalKeyColumn(self._jdf) - return col and Column(col) - - def tileLayerMetadata(self): - """ - Fetch the tile layer metadata. - :return: A dictionary of metadata. - """ - import json - return json.loads(str(self._jrfctx.tileLayerMetadata(self._jdf))) - - def spatialJoin(self, other_df): - """ - Spatially join this RasterFrame to the given RasterFrame. - :return: Joined RasterFrame. - """ - ctx = SparkContext._active_spark_context._rf_context - df = ctx._jrfctx.spatialJoin(self._jdf, other_df._jdf) - return RasterFrame(df, ctx._spark_session) - - def toIntRaster(self, colname, cols, rows): - """ - Convert a tile to an Int raster - :return: array containing values of the tile's cells - """ - resArr = self._jrfctx.toIntRaster(self._jdf, colname, cols, rows) - return resArr - - def toDoubleRaster(self, colname, cols, rows): - """ - Convert a tile to an Double raster - :return: array containing values of the tile's cells - """ - resArr = self._jrfctx.toDoubleRaster(self._jdf, colname, cols, rows) - return resArr - - def withBounds(self): - """ - Add a column called "bounds" containing the extent of each row. - :return: RasterFrame with "bounds" column. - """ - ctx = SparkContext._active_spark_context._rf_context - df = ctx._jrfctx.withBounds(self._jdf) - return RasterFrame(df, ctx._spark_session) - - def withCenter(self): - """ - Add a column called "center" containing the center of the extent of each row. - :return: RasterFrame with "center" column. - """ - ctx = SparkContext._active_spark_context._rf_context - df = ctx._jrfctx.withCenter(self._jdf) - return RasterFrame(df, ctx._spark_session) - - def withCenterLatLng(self): - """ - Add a column called "center" containing the center of the extent of each row in Lat Long form. - :return: RasterFrame with "center" column. - """ - ctx = SparkContext._active_spark_context._rf_context - df = ctx._jrfctx.withCenterLatLng(self._jdf) - return RasterFrame(df, ctx._spark_session) - - def withSpatialIndex(self): - """ - Add a column containing the spatial index of each row. - :return: RasterFrame with "center" column. - """ - ctx = SparkContext._active_spark_context._rf_context - df = ctx._jrfctx.withSpatialIndex(self._jdf) - return RasterFrame(df, ctx._spark_session) - - -class RasterSourceUDT(UserDefinedType): - @classmethod - def sqlType(self): - return StructType([ - StructField("raster_source_kryo", BinaryType(), False)]) - - @classmethod - def module(cls): - return 'pyrasterframes.types' - - @classmethod - def scalaUDT(cls): - return 'org.apache.spark.sql.rf.RasterSourceUDT' - - def serialize(self, obj): - if (obj is None): return None - return None - - def deserialize(self, datum): - return None - -class TileUDT(UserDefinedType): - @classmethod - def sqlType(self): - return StructType([ - StructField("cell_type", StringType(), False), - StructField("cols", ShortType(), False), - StructField("rows", ShortType(), False), - StructField("cells", BinaryType(), True), - StructField("ref", StructType([ - StructField("source", RasterSourceUDT(), False), - StructField("subextent", StructType([ - StructField("xmin", DoubleType(), False), - StructField("ymin", DoubleType(), False), - StructField("xmax", DoubleType(), False), - StructField("ymax", DoubleType(), False)]), True)]), True)]) - - @classmethod - def module(cls): - return 'pyrasterframes.types' - - @classmethod - def scalaUDT(cls): - return 'org.apache.spark.sql.rf.TileUDT' - - # NB: These will need implementations if UDFs are to be supported, - # preferably in numpy arrays. - def serialize(self, obj): - if (obj is None): return None - return None - - def deserialize(self, datum): - return None - - - -class TileExploder(JavaTransformer, JavaMLReadable, JavaMLWritable): - """ - Python wrapper for TileExploder.scala - """ - def __init__(self): - super(TileExploder, self).__init__() - self._java_obj = self._new_java_obj("astraea.spark.rasterframes.ml.TileExploder", self.uid) - -class NoDataFilter(JavaTransformer, JavaMLReadable, JavaMLWritable): - """ - Python wrapper for NoDataFilter.scala - """ - def __init__(self): - super(NoDataFilter, self).__init__() - self._java_obj = self._new_java_obj("astraea.spark.rasterframes.ml.NoDataFilter", self.uid) - def setInputCols(self, values): - self._java_obj.setInputCols(values) diff --git a/pyrasterframes/python/setup.cfg b/pyrasterframes/python/setup.cfg deleted file mode 100644 index d8fb59953..000000000 --- a/pyrasterframes/python/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ - -[aliases] -test=pytest - -[tool:pytest] -addopts = --verbose -testpaths = tests -python_files = *.py \ No newline at end of file diff --git a/pyrasterframes/python/setup.py b/pyrasterframes/python/setup.py deleted file mode 100644 index b24dfa3ad..000000000 --- a/pyrasterframes/python/setup.py +++ /dev/null @@ -1,155 +0,0 @@ -""" -The operations in this file are designed for development and testing only. -""" - - -from setuptools import setup, find_packages -import distutils.log -import importlib - - -def _extract_module(mod): - module = importlib.import_module(mod) - - if hasattr(module, '__all__'): - globals().update({n: getattr(module, n) for n in module.__all__}) - else: - globals().update({k: v for (k, v) in module.__dict__.items() if not k.startswith('_')}) - - - -class ExampleCommand(distutils.cmd.Command): - """A custom command to run pyrasterframes examples.""" - - description = 'run pyrasterframes examples' - user_options = [ - # The format is (long option, short option, description). - ('examples=', 'e', 'examples to run'), - ] - - def initialize_options(self): - from pathlib import Path - """Set default values for options.""" - # Each user option must be listed here with their default value. - self.examples = filter(lambda x: not x.name.startswith('_'), - list(Path('./examples').resolve().glob('*.py'))) - - def _check_ex_path(self, ex): - from pathlib import Path - file = Path(ex) - if not file.suffix: - file = file.with_suffix('.py') - file = (Path('./examples') / file).resolve() - - assert file.is_file(), ('Invalid example %s' % file) - return file - - def finalize_options(self): - """Post-process options.""" - import re - if isinstance(self.examples, str): - self.examples = re.split('\W+', self.examples) - self.examples = map(lambda x: 'examples.' + x.stem, - map(self._check_ex_path, self.examples)) - - - def run(self): - """Run the examples.""" - import traceback - for ex in self.examples: - print(('-' * 50) + '\nRunning %s' % ex + '\n' + ('-' * 50)) - try: - _extract_module(ex) - except Exception: - print(('-' * 50) + '\n%s Failed:' % ex + '\n' + ('-' * 50)) - print(traceback.format_exc()) - - -class ZipCommand(distutils.cmd.Command): - """A custom command to create a minimal zip distribution.""" - - description = 'create a minimal pyrasterframes source zip file' - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - """Create the zip.""" - import zipfile - from pathlib import Path - import os - zfile = 'pyrasterframes.zip' - - if os.path.isfile(zfile): - os.remove(zfile) - with zipfile.ZipFile(zfile, 'w') as przip: - przip.write('pyrasterframes') - # Bring in source files and readme - patterns = ['*.py', '*.rst', '*.jar'] - root = Path('.').resolve() - for pattern in patterns: - for file in list(root.glob('pyrasterframes/' + pattern)): - przip.write(str(file.relative_to(root))) - # Put a copy of the license in the zip - przip.write('LICENSE.md', 'pyrasterframes/LICENSE.md') - - - - - -with open('README.rst') as f: - readme = f.read() - -#pyspark_ver = 'pyspark>=2.2.0,<2.3' -pyspark_ver = 'pyspark==2.3.2' - -setup_args = dict( - name='pyrasterframes', - description='Python bindings for RasterFrames', - long_description=readme, - version='0.0.1', - url='http://rasterframes.io', - author='D. Benjamin Guseman', - author_email='guseman@astraea.io', - license='Apache 2', - setup_requires=['pytest-runner', pyspark_ver, 'pathlib'], - install_requires=[ - # pyspark_ver, - # 'pathlib' - ], - tests_require=[ - pyspark_ver, - 'pytest==3.4.2', - 'pypandoc', - 'numpy>=1.7' - ], - test_suite="pytest-runner", - packages=find_packages(exclude=['tests', 'examples']), - include_package_data=True, - package_data={'.':['LICENSE.md'], 'pyrasterframes':['*.jar']}, - exclude_package_data={'.':['setup.*', 'README.*']}, - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Environment :: Other Environment', - 'License :: OSI Approved :: Apache Software License', - 'Natural Language :: English', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries' - ], - zip_safe=False, - cmdclass={ - 'examples': ExampleCommand, - 'minzip': ZipCommand - } - # entry_points={ - # "console_scripts": ['pyrasterframes=pyrasterframes:console'] - # } -) - -if __name__ == "__main__": - setup(**setup_args) diff --git a/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff deleted file mode 100644 index 0d5a644ea..000000000 Binary files a/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff deleted file mode 100644 index 947cef3a6..000000000 Binary files a/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff deleted file mode 100644 index f57004679..000000000 Binary files a/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff deleted file mode 100644 index b287e5180..000000000 Binary files a/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff deleted file mode 100644 index 61a95250c..000000000 Binary files a/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff deleted file mode 100644 index 2534d4bd0..000000000 Binary files a/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff deleted file mode 100644 index 9a574fce8..000000000 Binary files a/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff deleted file mode 100644 index 1a1ad3c7f..000000000 Binary files a/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff deleted file mode 100644 index 56616e0f6..000000000 Binary files a/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff deleted file mode 100644 index 9bbe2c89f..000000000 Binary files a/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff b/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff deleted file mode 100644 index 224ec5ac9..000000000 Binary files a/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff and /dev/null differ diff --git a/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff deleted file mode 100644 index 3f596e19f..000000000 Binary files a/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff and /dev/null differ diff --git a/pyrasterframes/python/tests/PyRasterFramesTests.py b/pyrasterframes/python/tests/PyRasterFramesTests.py deleted file mode 100644 index b47e25577..000000000 --- a/pyrasterframes/python/tests/PyRasterFramesTests.py +++ /dev/null @@ -1,234 +0,0 @@ - -from pyspark.sql import SparkSession, Column -from pyspark.sql.functions import * -from pyrasterframes import * -from pyrasterframes.rasterfunctions import * -from pathlib import Path -import os -import unittest - -# version-conditional imports -import sys -if sys.version_info[0] > 2: - import builtins -else: - import __builtin__ as builtins - - -def _rounded_compare(val1, val2): - print('Comparing {} and {} using round()'.format(val1, val2)) - return builtins.round(val1) == builtins.round(val2) - - -class RasterFunctionsTest(unittest.TestCase): - - - @classmethod - def setUpClass(cls): - - # gather Scala requirements - jarpath = list(Path('../target/scala-2.11').resolve().glob('pyrasterframes-assembly*.jar'))[0] - - # hard-coded relative path for resources - cls.resource_dir = Path('./static').resolve() - - # spark session with RF - cls.spark = (SparkSession.builder - .config('spark.driver.extraClassPath', jarpath) - .config('spark.executor.extraClassPath', jarpath) - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - .config("spark.kryo.registrator", "astraea.spark.rasterframes.util.RFKryoRegistrator") - .config("spark.kryoserializer.buffer.max", "500m") - .getOrCreate()) - cls.spark.sparkContext.setLogLevel('ERROR') - print(cls.spark.version) - cls.spark.withRasterFrames() - - # load something into a rasterframe - rf = cls.spark.read.geotiff(cls.resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri()) \ - .withBounds() \ - .withCenter() - - # convert the tile cell type to provide for other operations - cls.tileCol = 'tile' - cls.rf = rf.withColumn('tile2', convert_cell_type(cls.tileCol, 'float32')) \ - .drop(cls.tileCol) \ - .withColumnRenamed('tile2', cls.tileCol).asRF() - #cls.rf.show() - - - def test_identify_columns(self): - cols = self.rf.tileColumns() - self.assertEqual(len(cols), 1, '`tileColumns` did not find the proper number of columns.') - print("Tile columns: ", cols) - col = self.rf.spatialKeyColumn() - self.assertIsInstance(col, Column, '`spatialKeyColumn` was not found') - print("Spatial key column: ", col) - col = self.rf.temporalKeyColumn() - self.assertIsNone(col, '`temporalKeyColumn` should be `None`') - print("Temporal key column: ", col) - - - def test_tile_operations(self): - df1 = self.rf.withColumnRenamed(self.tileCol, 't1').asRF() - df2 = self.rf.withColumnRenamed(self.tileCol, 't2').asRF() - df3 = df1.spatialJoin(df2).asRF() - df3 = df3.withColumn('norm_diff', normalized_difference('t1', 't2')) - df3.printSchema() - - aggs = df3.agg( - agg_mean('norm_diff'), - ) - aggs.show() - row = aggs.first() - - self.assertTrue(_rounded_compare(row['agg_mean(norm_diff)'], 0)) - - - def test_general(self): - meta = self.rf.tileLayerMetadata() - self.assertIsNotNone(meta['bounds']) - df = self.rf.withColumn('dims', tile_dimensions(self.tileCol)) \ - .withColumn('type', cell_type(self.tileCol)) \ - .withColumn('dCells', data_cells(self.tileCol)) \ - .withColumn('ndCells', no_data_cells(self.tileCol)) \ - .withColumn('min', tile_min(self.tileCol)) \ - .withColumn('max', tile_max(self.tileCol)) \ - .withColumn('mean', tile_mean(self.tileCol)) \ - .withColumn('sum', tile_sum(self.tileCol)) \ - .withColumn('stats', tile_stats(self.tileCol)) \ - .withColumn('envelope', envelope('bounds')) \ - .withColumn('ascii', render_ascii(self.tileCol)) \ - .withColumn('log', log(self.tileCol)) \ - .withColumn('exp', exp(self.tileCol)) \ - .withColumn('expm1', expm1(self.tileCol)) \ - .withColumn('round', round(self.tileCol)) - - df.show() - - def test_rasterize(self): - # NB: This test just makes sure rasterize runs, not that the results are correct. - withRaster = self.rf.withColumn('rasterize', rasterize('bounds', 'bounds', lit(42), 10, 10)) - withRaster.show() - - def test_reproject(self): - reprojected = self.rf.withColumn('reprojected', reproject_geometry('center', 'EPSG:4326', 'EPSG:3857')) - reprojected.show() - - def test_aggregations(self): - aggs = self.rf.agg( - agg_mean(self.tileCol), - agg_data_cells(self.tileCol), - agg_no_data_cells(self.tileCol), - agg_stats(self.tileCol), - agg_approx_histogram(self.tileCol) - ) - aggs.show() - row = aggs.first() - - self.assertTrue(_rounded_compare(row['agg_mean(tile)'], 10160)) - print(row['agg_data_cells(tile)']) - self.assertEqual(row['agg_data_cells(tile)'], 387000) - self.assertEqual(row['agg_no_data_cells(tile)'], 1000) - self.assertEqual(row['agg_stats(tile)'].data_cells, row['agg_data_cells(tile)']) - - - def test_sql(self): - - self.rf.createOrReplaceTempView("rf") - - dims = self.rf.withColumn('dims', tile_dimensions(self.tileCol)).first().dims - dims_str = """{}, {}""".format(dims.cols, dims.rows) - - self.spark.sql("""SELECT tile, rf_make_constant_tile(1, {}, 'uint16') AS One, - rf_make_constant_tile(2, {}, 'uint16') AS Two FROM rf""".format(dims_str, dims_str)) \ - .createOrReplaceTempView("r3") - - ops = self.spark.sql("""SELECT tile, rf_local_add(tile, One) AS AndOne, - rf_local_subtract(tile, One) AS LessOne, - rf_local_multiply(tile, Two) AS TimesTwo, - rf_local_divide(tile, Two) AS OverTwo - FROM r3""") - - ops.printSchema - statsRow = ops.select(tile_mean(self.tileCol).alias('base'), - tile_mean("AndOne").alias('plus_one'), - tile_mean("LessOne").alias('minus_one'), - tile_mean("TimesTwo").alias('double'), - tile_mean("OverTwo").alias('half')) \ - .first() - - self.assertTrue(_rounded_compare(statsRow.base, statsRow.plus_one - 1)) - self.assertTrue(_rounded_compare(statsRow.base, statsRow.minus_one + 1)) - self.assertTrue(_rounded_compare(statsRow.base, statsRow.double / 2)) - self.assertTrue(_rounded_compare(statsRow.base, statsRow.half * 2)) - - def test_explode(self): - import pyspark.sql.functions as F - self.rf.select('spatial_key', explode_tiles(self.tileCol)).show() - # +-----------+------------+---------+-------+ - # |spatial_key|column_index|row_index|tile | - # +-----------+------------+---------+-------+ - # |[2,1] |4 |0 |10150.0| - cell = self.rf.select(self.rf.spatialKeyColumn(), explode_tiles(self.rf.tile)) \ - .where(F.col("spatial_key.col")==2) \ - .where(F.col("spatial_key.row")==1) \ - .where(F.col("column_index")==4) \ - .where(F.col("row_index")==0) \ - .select(F.col("tile")) \ - .collect()[0][0] - self.assertEqual(cell, 10150.0) - - # Test the sample version - frac = 0.01 - sample_count = self.rf.select(explode_tiles_sample(frac, 1872, self.tileCol)).count() - print('Sample count is {}'.format(sample_count)) - self.assertTrue(sample_count > 0) - self.assertTrue(sample_count < (frac * 1.1) * 387000) # give some wiggle room - - - def test_maskByValue(self): - from pyspark.sql.functions import lit - - # create an artificial mask for values > 25000; masking value will be 4 - mask_value = 4 - - rf1 = self.rf.select(self.rf.tile, - local_multiply( - convert_cell_type( - local_greater_scalar_int(self.rf.tile, 25000), - "uint8"), - lit(mask_value)).alias('mask')) - rf2 = rf1.select(rf1.tile, mask_by_value(rf1.tile, rf1.mask, lit(mask_value)).alias('masked')) - result = rf2.agg(agg_no_data_cells(rf2.tile) < agg_no_data_cells(rf2.masked)) \ - .collect()[0][0] - self.assertTrue(result) - - - def test_resample(self): - from pyspark.sql.functions import lit - result = self.rf.select( - tile_min(local_equal( - resample(resample(self.rf.tile, lit(2)), lit(0.5)), - self.rf.tile)) - ).collect()[0][0] - - self.assertTrue(result == 1) # short hand for all values are true - - -def suite(): - functionTests = unittest.TestSuite() - functionTests.addTest(RasterFunctionsTest('test_identify_columns')) - functionTests.addTest(RasterFunctionsTest('test_tile_operations')) - functionTests.addTest(RasterFunctionsTest('test_general')) - functionTests.addTest(RasterFunctionsTest('test_rasterize')) - functionTests.addTest(RasterFunctionsTest('test_reproject')) - functionTests.addTest(RasterFunctionsTest('test_aggregations')) - functionTests.addTest(RasterFunctionsTest('test_explode')) - functionTests.addTest(RasterFunctionsTest('test_sql')) - functionTests.addTest(RasterFunctionsTest('test_maskByValue')) - functionTests.addTest(RasterFunctionsTest('test_resample')) - return functionTests - - -unittest.TextTestRunner().run(suite()) diff --git a/pyrasterframes/src/main/python/.gitignore b/pyrasterframes/src/main/python/.gitignore new file mode 100644 index 000000000..d43a8f7ce --- /dev/null +++ b/pyrasterframes/src/main/python/.gitignore @@ -0,0 +1,3 @@ +.coverage +htmlcov + diff --git a/pyrasterframes/python/LICENSE.md b/pyrasterframes/src/main/python/LICENSE.txt similarity index 82% rename from pyrasterframes/python/LICENSE.md rename to pyrasterframes/src/main/python/LICENSE.txt index 0dd8f3f8c..e7ca21e70 100644 --- a/pyrasterframes/python/LICENSE.md +++ b/pyrasterframes/src/main/python/LICENSE.txt @@ -1,12 +1,12 @@ This software is licensed under the Apache 2 license, quoted below. -Copyright 2017-2018 Astraea. Inc. +Copyright 2017-2019 Astraea, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - [http://www.apache.org/licenses/LICENSE-2.0] +[http://www.apache.org/licenses/LICENSE-2.0] Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT @@ -14,3 +14,4 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +SPDX-License-Identifier: Apache-2.0 diff --git a/pyrasterframes/src/main/python/MANIFEST.in b/pyrasterframes/src/main/python/MANIFEST.in new file mode 100644 index 000000000..68903bdfe --- /dev/null +++ b/pyrasterframes/src/main/python/MANIFEST.in @@ -0,0 +1,3 @@ + +global-exclude *.py[cod] __pycache__ .DS_Store +recursive-include deps/jars *.jar diff --git a/pyrasterframes/src/main/python/README.md b/pyrasterframes/src/main/python/README.md new file mode 100644 index 000000000..22a6f30c7 --- /dev/null +++ b/pyrasterframes/src/main/python/README.md @@ -0,0 +1,43 @@ +# PyRasterFrames + +PyRasterFrames enables access and processing of geospatial raster data in PySpark DataFrames. + +## Getting started + +The quickest way to get started is to [`pip`](https://pip.pypa.io/en/stable/installing/) install the pyrasterframes package. + +```bash +pip install pyrasterframes +``` + +You can then access a [`pyspark SparkSession`](https://spark.apache.org/docs/latest/api/python/pyspark.sql.html#pyspark.sql.SparkSession) using the [`local[*]` master](https://spark.apache.org/docs/latest/submitting-applications.html#master-urls) in your python interpreter as follows. + +```python +import pyrasterframes +spark = pyrasterframes.get_spark_session() +``` + +Then you can read a raster and do some work with it. + +```python +from pyrasterframes.rasterfunctions import * +from pyspark.sql.functions import lit +# Read a MODIS surface reflectance granule +df = spark.read.raster('https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF') +# Add 3 element-wise, show some rows of the dataframe +df.select(rf_local_add(df.tile, lit(3))).show(5, False) +``` + +## Support + +Reach out to us on [gitter][gitter]! + +Issue tracking is through [github](https://github.com/locationtech/rasterframes/issues). + +## Contributing + +Community contributions are always welcome. To get started, please review our [contribution guidelines](https://github.com/locationtech/rasterframes/blob/develop/CONTRIBUTING.md), [code of conduct](https://github.com/locationtech/rasterframes/blob/develop/CODE_OF_CONDUCT.md), and [developer's guide](../../../README.md). Reach out to us on [gitter][gitter] so the community can help you get started! + + + +[gitter]: https://gitter.im/locationtech/rasterframes diff --git a/pyrasterframes/src/main/python/deps/jars/README.md b/pyrasterframes/src/main/python/deps/jars/README.md new file mode 100644 index 000000000..693a02cb2 --- /dev/null +++ b/pyrasterframes/src/main/python/deps/jars/README.md @@ -0,0 +1,4 @@ +# pyrasterframes.jars + +Submodule containing JARs needed for pyrasterframe + diff --git a/pyrasterframes/src/main/python/docs/__init__.py b/pyrasterframes/src/main/python/docs/__init__.py new file mode 100644 index 000000000..e0aee3a26 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/__init__.py @@ -0,0 +1,51 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from pyspark.sql import DataFrame +from pyrasterframes import RFContext +from pweave import PwebPandocFormatter + + +def docs_dir(): + return os.path.dirname(os.path.realpath(__file__)) + + +# This is temporary until we port to run on web assets. +def resource_dir(): + # we may consider using gitpython which I think would be appropriate in the context of building docs + # see https://stackoverflow.com/a/41920796/2787937 + here = docs_dir() + test_resource = os.path.realpath(os.path.join(here, '..', '..', '..', 'src', 'test', 'resources')) + + return test_resource + + +def resource_dir_uri(): + return 'file://' + resource_dir() + + +class PegdownMarkdownFormatter(PwebPandocFormatter): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Pegdown doesn't support the width and label options. + def make_figure_string(self, figname, width, label, caption=""): + return "![%s](%s)" % (caption, figname) diff --git a/pyrasterframes/src/main/python/docs/aggregation.pymd b/pyrasterframes/src/main/python/docs/aggregation.pymd new file mode 100644 index 000000000..586458bcf --- /dev/null +++ b/pyrasterframes/src/main/python/docs/aggregation.pymd @@ -0,0 +1,143 @@ +# Aggregation + +```python, setup, echo=False +from pyrasterframes import rf_ipython +from docs import * +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import * +from pyspark.sql import * +import os + +spark = create_rf_spark_session() +``` + +There are three types of aggregate functions: _tile_ aggregate, DataFrame aggregate, and element-wise local aggregate. In the @ref:[tile aggregate functions](reference.md#tile-statistics), we are computing a statistical summary per row of a _tile_ column in a DataFrame. In the @ref:[DataFrame aggregate functions](reference.md#aggregate-tile-statistics), we are computing statistical summaries over all of the cell values *and* across all of the rows in the DataFrame or group. In the @ref:[element-wise local aggregate functions](reference.md#tile-local-aggregate-statistics), we are computing the element-wise statistical summary across a DataFrame or group of _tiles_. + +## Tile Mean Example + +We can illustrate aggregate differences by computing an aggregate mean. First, we create a sample DataFrame of 2 _tiles_ where the first _tile_ is composed of 25 values of 1.0 and the second _tile_ is composed of 25 values of 3.0. + +```python, sql_dataframe +import pyspark.sql.functions as F + +rf = spark.sql(""" +SELECT 1 as id, rf_make_ones_tile(5, 5, 'float32') as tile +UNION +SELECT 2 as id, rf_local_multiply(rf_make_ones_tile(5, 5, 'float32'), 3) as tile +""") + +tiles = rf.select("tile").collect() +print(tiles[0]['tile'].cells) +print(tiles[1]['tile'].cells) +``` + +We use the @ref:[`rf_tile_mean`](reference.md#rf-tile-mean) function to compute the _tile_ aggregate mean of cells in each row of column `tile`. The mean of each _tile_ is computed separately, so the first mean is 1.0 and the second mean is 3.0. Notice that the number of rows in the DataFrame is the same before and after the aggregation. + +```python, tile_mean, results='raw' +rf.select(F.col('id'), rf_tile_mean(F.col('tile'))).show() +``` + +We use the @ref:[`rf_agg_mean`](reference.md#rf-agg-mean) function to compute the DataFrame aggregate, which averages 25 values of 1.0 and 25 values of 3.0, across the fifty cells in two rows. Note that only a single row is returned since the average is computed over the full DataFrame. + +```python, agg_mean, results='raw' +rf.agg(rf_agg_mean(F.col('tile'))).show() +``` + +We use the @ref:[`rf_agg_local_mean`](reference.md#rf-agg-local-mean) function to compute the element-wise local aggregate mean across the two rows. For this aggregation, we are computing the mean of one value of 1.0 and one value of 3.0 to arrive at the element-wise mean, but doing so twenty-five times, one for each position in the _tile_. + +To compute an element-wise local aggregate, _tiles_ need to have the same dimensions. In this case, both _tiles_ have 5 rows and 5 columns. If we tried to compute an element-wise local aggregate over the DataFrame without equal _tile_ dimensions, we would get a runtime error. + +```python, local_mean +t = rf.agg(rf_agg_local_mean(F.col('tile')).alias('local_mean')) \ + .collect()[0]['local_mean'] +print(t.cells) +``` + +## Cell Counts Example + +We can also count the total number of data and NoData cells over all the _tiles_ in a DataFrame using @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells) and @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells). There are ~3.8 million data cells and ~1.9 million NoData cells in this DataFrame. See the section on @ref:["NoData" handling](nodata-handling.md) for additional discussion on handling missing data. + +```python, cell_counts, results='raw' +rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF') +stats = rf.agg(rf_agg_data_cells('proj_raster'), rf_agg_no_data_cells('proj_raster')) + +stats.show() +``` + +## Statistical Summaries + +The statistical summary functions return a summary of cell values: number of data cells, number of NoData cells, minimum, maximum, mean, and variance, which can be computed as a _tile_ aggregate, a DataFrame aggregate, or an element-wise local aggregate. + +The @ref:[`rf_tile_stats`](reference.md#rf-tile-stats) function computes summary statistics separately for each row in a _tile_ column as shown below. + +```python, tile_stats +rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif') +stats = rf.select(rf_tile_stats('proj_raster').alias('stats')) + +stats.printSchema() +``` + +```python, show_stats, results='raw' +stats.select('stats.min', 'stats.max', 'stats.mean', 'stats.variance').show(10, truncate=False) +``` + +The @ref:[`rf_agg_stats`](reference.md#rf-agg-stats) function aggregates over all of the _tiles_ in a DataFrame and returns a statistical summary of all cell values as shown below. + +```python, agg_stats, results='raw' +rf.agg(rf_agg_stats('proj_raster').alias('stats')) \ + .select('stats.min', 'stats.max', 'stats.mean', 'stats.variance') \ + .show() +``` + +The @ref:[`rf_agg_local_stats`](reference.md#rf-agg-local-stats) function computes the element-wise local aggregate statistical summary as shown below. The DataFrame used in the previous two code blocks has unequal _tile_ dimensions, so a different DataFrame is used in this code block to avoid a runtime error. + +```python, agg_local_stats +rf = spark.sql(""" +SELECT 1 as id, rf_make_ones_tile(5, 5, 'float32') as tile +UNION +SELECT 2 as id, rf_make_constant_tile(3, 5, 5, 'float32') as tile +UNION +SELECT 3 as id, rf_make_constant_tile(5, 5, 5, 'float32') as tile +""").agg(rf_agg_local_stats('tile').alias('stats')) + +agg_local_stats = rf.select('stats.min', 'stats.max', 'stats.mean', 'stats.variance').collect() + +for r in agg_local_stats: + for stat in r.asDict(): + print(stat, ':\n', r[stat], '\n') +``` + +## Histogram + +The @ref:[`rf_tile_histogram`](reference.md#rf-tile-histogram) function computes a count of cell values within each row of _tile_ and outputs a `bins` array with the schema below. In the graph below, we have plotted each bin's `value` on the x-axis and `count` on the y-axis for the _tile_ in the first row of the DataFrame. + + +```python, tile_histogram +import matplotlib.pyplot as plt + +rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF') + +hist_df = rf.select(rf_tile_histogram('proj_raster')['bins'].alias('bins')) +hist_df.printSchema() + +bins_row = hist_df.first() +values = [int(bin['value']) for bin in bins_row.bins] +counts = [int(bin['count']) for bin in bins_row.bins] + +plt.hist(values, weights=counts, bins=100) +plt.show() +``` + +The @ref:[`rf_agg_approx_histogram`](reference.md#rf-agg-approx-histogram) function computes a count of cell values across all of the rows of _tile_ in a DataFrame or group. In the example below, the range of the y-axis is significantly wider than the range of the y-axis on the previous histogram since this histogram was computed for all cell values in the DataFrame. + + +```python, agg_histogram +bins_list = rf.agg( + rf_agg_approx_histogram('proj_raster')['bins'].alias('bins') + ).collect() +values = [int(row['value']) for row in bins_list[0].bins] +counts = [int(row['count']) for row in bins_list[0].bins] + +plt.hist(values, weights=counts, bins=100) +plt.show() +``` diff --git a/pyrasterframes/src/main/python/docs/concepts.md b/pyrasterframes/src/main/python/docs/concepts.md new file mode 100644 index 000000000..dda457055 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/concepts.md @@ -0,0 +1,71 @@ +# Concepts + +There are a number of Earth-observation (EO) concepts that crop up in the discussion of RasterFrames features. We'll cover these briefly in the sections below. However, here are a few links providing a more extensive introduction to working with Earth observation data. + +* [_Fundamentals of Remote Sensing_](https://www.nrcan.gc.ca/maps-tools-and-publications/satellite-imagery-and-air-photos/tutorial-fundamentals-remote-sensing/9309) +* [_Newcomers Earth Observation Guide_](https://business.esa.int/newcomers-earth-observation-guide) +* [_Earth Observation Markets and Applications_](https://www.ofcom.org.uk/__data/assets/pdf_file/0021/82047/introduction_eo_for_ofcom_june_2015_no_video.pdf) + +## Raster + +A raster is a regular grid of numeric values. A raster can be thought of as an image, as is the case if the values in the grid represent brightness along a greyscale. More generally, a raster can measure many different phenomena or encode a variety of different discrete classifications. + +## Cell + +A cell is a single row and column intersection in the raster grid. It is a single pixel in an image. A cell's value often represents one sample from a sensor encoded as a scalar value associated with a specific location and time. + +## Cell Type + +A numeric cell value may be encoded in a number of different computer numeric formats. There are typically three characteristics used to describe a cell type: + +* word size (bit-width) +* signed vs unsigned +* integral vs floating-point + + +The most frequently encountered cell types in RasterFrames are below. + +| Name | Abbreviation | Description | Range | +| --- | --- | --- | --- | +| Byte | `int8` | Signed 8-bit integral | -128 to 127 | +| Unsigned Byte | `uint8` | Unsigned 8-bit integral | 0 to 255 | +| Short | `int16` | Signed 16-bit integral | -32,768 to 32,767 | +| Unsigned Short | `uint16` | Unsigned 16-bit integral | 0 to 65,535 | +| Int | `int32` | Signed 32-bit integral | -2,147,483,648 to 2,147,483,647 | +| Unsigned Int | `uint32` | Unsigned 32-bit integral | 0 to 4,294,967,295 | +| Float | `float32` | 32-bit floating-point | -3.4028235E38 to 3.4028235E38 | +| Double | `float64` | 64-bit floating-point | -1.7976931348623157E308 to 1.7976931348623157E308 | + +See the section on [“NoData” Handling](nodata-handling.md) for additional discussion on cell types and more exhaustive coverage of available cell types. + +## NoData + +A "NoData" (or N/A) value is a specifically identified value for a cell type used to indicate the absence of data. See the section on @ref:[“NoData” Handling](nodata-handling.md) for additional discussion on "NoData". + +## Scene + +A scene (or granule) is a discrete instance of EO @ref:[raster data](concepts.md#raster) with a specific extent (region), date-time, and map projection (or CRS). + +## Band + +A @ref:[scene](concepts.md#scene) frequently defines many different measurements captured at the same date-time, over the same extent, and meant to be processed together. These different measurements are referred to as bands. The name comes from the varying bandwidths of light and electromagnetic radiation measured in many EO datasets. + +## Coordinate Reference System (CRS) + +A [coordinate reference system (or spatial reference system)][CRS] is a set of mathematical constructs used to translate locations on the three-dimensional surface of the earth to the two dimensional raster grid. A CRS typically accompanies any EO data so it can be precisely located. + +## Extent + +An extent (or bounding box) is a rectangular region specifying the geospatial coverage of a @ref:[raster](concepts.md#raster) or @ref:[tile](concepts.md#tile), a two-dimensional array of @ref:[cells](concepts.md#cell) within a single CRS. + +## Tile + +A tile (sometimes called a "chip") is a rectangular subset of a @ref:[scene](concepts.md#scene). As a scene is a raster, a tile is also a raster. A tile can conceptually be thought of as a two-dimensional array. + +Some EO data has many @ref:[bands](concepts.md#band) or channels. Within RasterFrames, this third dimension is handled across columns of the DataFrame, such that the tiles within DataFrames are all two-dimensional arrays. + +Tiles are often square and the dimensions are some power of two, for example 256 by 256. + +The tile is the primary discretization unit used in RasterFrames. The scene's overall @ref:[extent](concepts.md#extent) is carved up into smaller extents and spread across rows. + +[CRS]: https://en.wikipedia.org/wiki/Spatial_reference_system diff --git a/pyrasterframes/src/main/python/docs/description.md b/pyrasterframes/src/main/python/docs/description.md new file mode 100644 index 000000000..8db6347af --- /dev/null +++ b/pyrasterframes/src/main/python/docs/description.md @@ -0,0 +1,46 @@ +# Overview + +RasterFrames® provides a DataFrame-centric view over arbitrary Earth-observation (EO) data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of [Apache Spark](https://spark.apache.org/docs/latest/) [ML](https://spark.apache.org/docs/latest/ml-guide.html) algorithms. It provides APIs in @ref:[Python, SQL, and Scala](languages.md), and can scale from a laptop computer to a large distributed cluster, enabling _global_ analysis with satellite imagery in a wholly new, flexible, and convenient way. + +## Context + +We have a millennia-long history of organizing information in tabular form. Typically, rows represent independent events or observations, and columns represent attributes and measurements from the observations. The forms have evolved, from hand-written agricultural records and transaction ledgers, to the advent of spreadsheets on the personal computer, and on to the creation of the _DataFrame_ data structure as found in [R Data Frames][R] and [Python Pandas][Pandas]. The table-oriented data structure remains a common and critical component of organizing data across industries, and—most importantly—it is the mental model employed by data scientists across diverse forms of modeling and analysis. + +The evolution of the DataFrame form has continued with [Spark SQL](https://spark.apache.org/docs/latest/sql-programming-guide.html), which brings DataFrames to the big data distributed compute space. Through several novel innovations, Spark SQL enables data scientists to work with DataFrames too large for the memory of a single computer. As suggested by the name, these DataFrames are manipulatable via standard SQL, as well as the more general-purpose programming languages Python, R, Java, and Scala. + +RasterFrames, an incubating Eclipse Foundation LocationTech project, brings together EO data access, cloud computing, and DataFrame-based data science. The recent explosion of EO data from public and private satellite operators presents both a huge opportunity and a huge challenge to the data analysis community. It is _Big Data_ in the truest sense, and its footprint is rapidly getting bigger. According to a World Bank document on assets for post-disaster situation awareness[^1]: + +> Of the 1,738 operational satellites currently orbiting the earth (as of 9/[20]17), 596 are earth observation satellites and 477 of these are non-military assets (i.e. available to civil society including commercial entities and governments for earth observation, according to the Union of Concerned Scientists). This number is expected to increase significantly over the next ten years. The 200 or so planned remote sensing satellites have a value of over 27 billion USD (Forecast International). This estimate does not include the burgeoning fleets of smallsats as well as micro, nano and even smaller satellites... All this enthusiasm has, not unexpectedly, led to a veritable fire-hose of remotely sensed data which is becoming difficult to navigate even for seasoned experts. + +## Benefit + +By using DataFrames as the core cognitive and compute data model for processing EO data, RasterFrames is able to deliver sophisticated computational and algorithmic capabilities in a tabular form that is familiar and accessible to the general computing public. Because it is built on Apache Spark, solutions prototyped on a laptop can be easily scaled to run on cluster and cloud compute resources. Apache Spark also provides integration between its DataFrame libraries and machine learning, with which RasterFrames is fully compatible. + +## Architecture + +RasterFrames builds upon several other LocationTech projects: +[GeoTrellis](https://geotrellis.io/), [GeoMesa](https://www.geomesa.org/), +[JTS](https://github.com/locationtech/jts), and +[SFCurve](https://github.com/locationtech/sfcurve). + +![LocationTech Stack](static/rasterframes-locationtech-stack.png) + +RasterFrames introduces georectified raster imagery to Spark SQL. It quantizes scenes into chunks called @ref:[_tiles_](concepts.md#tile). Each _tile_ contains a 2-D matrix of @ref:[_cell_](concepts.md#cell) or pixel values along with information on how to numerically interpret those _cells_. + +As shown in the figure below, a "RasterFrame" is a Spark DataFrame with one or more columns of type @ref:[_tile_](concepts.md#tile). A _tile_ column typically represents a single frequency band of sensor data, such as "blue" or "near infrared", but can also be quality assurance information, land classification assignments, or any other raster spatial data. Along with _tile_ columns there is typically an @ref:[`extent`](concepts.md#extent) specifying the geographic location of the data, the map projection of that geometry (@ref:[`crs`](concepts.md#coordinate-reference-system--crs-)), and a `timestamp` column representing the acquisition time. These columns can all be used in the `WHERE` clause when filtering. + +@@include[RasterFrame Example](static/rasterframe-sample.md) + +RasterFrames also includes support for working with vector data, such as [GeoJSON][GeoJSON]. RasterFrames vector data operations let you filter with geospatial relationships like contains or intersects, mask cells, convert vectors to rasters, and more. + +Raster data can be read from a @ref:[number of sources](raster-io.md). Through the flexible Spark SQL DataSource API, RasterFrames can be constructed from collections of imagery (including Cloud Optimized GeoTIFFs or [COGS][COGS]), [GeoTrellis Layers][GTLayer], and from catalogs of large datasets like Landsat 8 and MODIS data sets on the @ref:[AWS Public Data Set (PDS)](raster-catalogs.md#using-external-catalogs). + +[R]:https://www.rdocumentation.org/packages/base/versions/3.5.1/topics/data.frame +[Pandas]:https://pandas.pydata.org/ +[GeoJSON]:https://en.wikipedia.org/wiki/GeoJSON +[GTLayer]:https://geotrellis.readthedocs.io/en/latest/guide/core-concepts.html#layouts-and-tile-layers +[PDS]:https://registry.opendata.aws/modis/ +[COGS]:https://www.cogeo.org/ + +[^1]: [_Demystifying Satellite Assets for Post-Disaster Situation Awareness_](https://docs.google.com/document/d/11bIw5HcEiZy8SKli6ZFQC2chVEiiIJ-f0o6btA4LU48). +World Bank via [OpenDRI.org](https://opendri.org/resource/demystifying-satellite-assets-for-post-disaster-situation-awareness/). Accessed November 28, 2018. diff --git a/pyrasterframes/src/main/python/docs/getting-started.pymd b/pyrasterframes/src/main/python/docs/getting-started.pymd new file mode 100644 index 000000000..4051286ae --- /dev/null +++ b/pyrasterframes/src/main/python/docs/getting-started.pymd @@ -0,0 +1,149 @@ +# Getting Started + +@@@ note + +If you are new to Earth-observing imagery, you might consider looking at the [Concepts](concepts.md) section first. + +@@@ + +There are @ref:[several ways](getting-started.md#other-options) to use RasterFrames, and @ref:[several languages](languages.md) with which you can use it. + +The simplest way to get started with RasterFrames is via the the Python shell. To get started you will need: + +1. [Python](https://www.python.org/) installed. Version 3.6 or greater is recommended. +1. [`pip`](https://pip.pypa.io/en/stable/installing/) installed. If you are using Python 3, `pip` may already be installed. +1. Java [JDK 8](https://openjdk.java.net/install/index.html) installed on your system and `java` on your system `PATH` or `JAVA_HOME` pointing to a Java installation. + +## pip install pyrasterframes + +```bash +$ python3 -m pip install pyrasterframes +``` + +Then in a python interpreter of your choice, you can get a [`pyspark` `SparkSession`](https://spark.apache.org/docs/latest/api/python/pyspark.sql.html#pyspark.sql.SparkSession) using the [`local[*]` master](https://spark.apache.org/docs/latest/submitting-applications.html#master-urls). + + +```python, setup, echo=False +from pyrasterframes import rf_ipython +``` + +```python, version +import pyrasterframes +spark = pyrasterframes.get_spark_session() +``` + +Then, you can read a raster and work with it in a Spark DataFrame. + +```python, local_add, results='raw' +from pyrasterframes.rasterfunctions import * +from pyspark.sql.functions import lit + +# Read a MODIS surface reflectance granule +df = spark.read.raster('https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF') + +# Add 3 element-wise, show some rows of the DataFrame +df.withColumn('added', rf_local_add(df.proj_raster, lit(3))) \ + .select(rf_crs('added'), rf_extent('added'), rf_tile('added')) \ + .show(3) +``` + +This example is extended in the [getting started Jupyter notebook](https://nbviewer.jupyter.org/github/locationtech/rasterframes/blob/develop/rf-notebook/src/main/notebooks/Getting%20Started.ipynb). + +## Next Steps + +To understand more about how and why RasterFrames represents Earth observation in DataFrames, read about the @ref:[core concepts](concepts.md) and the project @ref:[description](description.md). For more hands-on examples, see the chapters about @ref:[reading](raster-io.md) and @ref:[processing](raster-processing.md) with RasterFrames. + +## Other Options + +You can also use RasterFrames in the following environments: + +1. Jupyter Notebook +1. `pyspark` shell + +### Using Jupyter Notebook + + RasterFrames provides a Docker image for a Jupyter notebook server whose default kernel is already set up for running RasterFrames. To use it: + + 1. Install [Docker](https://docs.docker.com/install/) + 1. Pull the image: `docker pull s22s/rasterframes-notebook` + 1. Run a container with the image, for example: + `docker run -p 8808:8888 -p 44040:4040 -v /path/to/notebooks:/home/jovyan/work rasterframes-notebook:latest` + 1. In a browser, open `localhost:8808` in the example above. + +See the [RasterFrames Notebook README](https://github.com/locationtech/rasterframes/blob/develop/rf-notebook/README.md) for instructions on building the Docker image for this Jupyter notebook server. + +### Using `pyspark` shell + +You can use RasterFrames in a `pyspark` shell. To set up the `pyspark` environment, prepare your call with the appropriate `--master` and other `--conf` arguments for your cluster manager and environment. For RasterFrames support you need to pass arguments pointing to the various Java dependencies. You will also need the Python source zip, even if you have pip installed the package. You can download the source zip here: https://repo1.maven.org/maven2/org/locationtech/rasterframes/pyrasterframes_2.11/${VERSION}/pyrasterframes_2.11-${VERSION}-python.zip. + +The `pyspark` shell command will look something like this. + +```bash + pyspark \ + --master local[*] \ + --py-files pyrasterframes_2.11-${VERSION}-python.zip \ + --packages org.locationtech.rasterframes:rasterframes_2.11:${VERSION},org.locationtech.rasterframes:pyrasterframes_2.11:${VERSION},org.locationtech.rasterframes:rasterframes-datasource_2.11:${VERSION} \ + --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ # these configs improve serialization performance + --conf spark.kryo.registrator=org.locationtech.rasterframes.util.RFKryoRegistrator \ + --conf spark.kryoserializer.buffer.max=500m +``` + +Then in the `pyspark` shell, import the module and call `withRasterFrames` on the SparkSession. + +```python, shell, evaluate=False +Welcome to + ____ __ + / __/__ ___ _____/ /__ + _\ \/ _ \/ _ `/ __/ '_/ + /__ / .__/\_,_/_/ /_/\_\ version 2.3.2 + /_/ + +Using Python version 3.7.3 (default, Mar 27 2019 15:43:19) +SparkSession available as 'spark'. +>>> import pyrasterframes +>>> spark = spark.withRasterFrames() +>>> df = spark.read.raster('https://landsat-pds.s3.amazonaws.com/c1/L8/158/072/LC08_L1TP_158072_20180515_20180604_01_T1/LC08_L1TP_158072_20180515_20180604_01_T1_B5.TIF') +``` + +Now you have the configured SparkSession with RasterFrames enabled. + +## Installing GDAL + +GDAL provides a wide variety of drivers to read data from many different raster formats. If GDAL is installed in the environment, RasterFrames will be able to @ref:[read](raster-read.md) those formats. If you are using the @ref:[Jupyter Notebook image](getting-started.md#jupyter-notebook), GDAL is already installed for you. Otherwise follow the instructions below. Version 2.4.1 or greater is required. + +### Installing on MacOS + +Using [homebrew](https://brew.sh/): + +```bash +brew install gdal +``` + +### Installing on Linux + +Using [`apt-get`](https://wiki.debian.org/Apt): + +```bash +sudo apt-get update +sudo apt-get install gdal-bin +``` + +### Testing For GDAL + +```bash +gdalinfo --formats +``` + +To support GeoTIFF and JPEG2000 formats, you should look for the following drivers from the output above: + +* `GTiff -raster- (rw+vs): GeoTIFF` +* `JPEG2000 -raster,vector- (rwv): JPEG-2000 part 1 (ISO/IEC 15444-1), based on Jasper library` + +Do the following to see if RasterFrames was able to find GDAL: + +```python, gdal_version, evaluate=False +from pyrasterframes.utils import gdal_version +print(gdal_version()) +``` + +This will print out something like "GDAL x.y.z, released 20yy/mm/dd". If it reports "not available", then GDAL is not installed in a place where the RasterFrames runtime was able to find it. Please [file an issue](https://github.com/locationtech/rasterframes/issues) to get help resolving this problem. diff --git a/pyrasterframes/src/main/python/docs/index.md b/pyrasterframes/src/main/python/docs/index.md new file mode 100644 index 000000000..d5f27a20b --- /dev/null +++ b/pyrasterframes/src/main/python/docs/index.md @@ -0,0 +1,52 @@ +# RasterFrames + +RasterFrames® brings together Earth-observation (EO) data access, cloud computing, and DataFrame-based data science. The recent explosion of EO data from public and private satellite operators presents both a huge opportunity and a huge challenge to the data analysis community. It is _Big Data_ in the truest sense, and its footprint is rapidly getting bigger. + +RasterFrames provides a DataFrame-centric view over arbitrary raster data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of Spark ML algorithms. By using DataFrames as the core cognitive and compute data model, it is able to deliver these features in a form that is both accessible to general analysts and scalable along with the rapidly growing data footprint. + +To learn more, please see the @ref:[Getting Started](getting-started.md) section of this manual. + +The source code can be found on GitHub at [locationtech/rasterframes](https://github.com/locationtech/rasterframes). + + + +
                          + +@@@ div { .md-left} + +## Detailed Contents + +@@ toc { depth=3 } + +@@@ + +@@@ div { .md-right } + +## Related Links + +* [Gitter Channel](https://gitter.im/locationtech/rasterframes) +* [Scala API Documentation](latest/api/index.html) +* [GitHub Repository](https://github.com/locationtech/rasterframes) +* [Astraea, Inc.](http://www.astraea.earth/), the company behind RasterFrames + +@@@ + +@@@ div { .md-clear } + +  + +@@@ + +@@@ index +* [Overview](description.md) +* [Getting Started](getting-started.md) +* [Concepts](concepts.md) +* [Raster Data I/O](raster-io.md) +* [Vector Data](vector-data.md) +* [Raster Processing](raster-processing.md) +* [Numpy and Pandas](numpy-pandas.md) +* [API Languages](languages.md) +* [Function Reference](reference.md) +* [Release Notes](release-notes.md) +@@@ + diff --git a/pyrasterframes/src/main/python/docs/languages.pymd b/pyrasterframes/src/main/python/docs/languages.pymd new file mode 100644 index 000000000..5111b7b74 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/languages.pymd @@ -0,0 +1,180 @@ +# API Languages + +One of the great powers of RasterFrames, afforded by Spark SQL, is the ability to express computation in multiple programming languages. This documentation focuses on Python because it is the most commonly used language in data science and GIS analytics. However, Scala (the implementation language of RasterFrames) and SQL are also fully supported. Examples in Python can be mechanically translated into the other two languages without much difficulty once the naming conventions are understood. In the sections below we will show the same example program in each language. We will compute the average NDVI per month for a single _tile_ in Tanzania. + +```python, imports, echo=False +from pyspark.sql.functions import * +from pyrasterframes.utils import create_rf_spark_session + +from pyrasterframes.rasterfunctions import * +import pyrasterframes.rf_ipython +import pandas as pd +import os +spark = create_rf_spark_session() +``` + +## Python + +### Step 1: Load the catalog + +```python, step_1_python +modis = spark.read.format('aws-pds-modis-catalog').load() +``` +### Step 2: Down-select data by month + +```python, step_2_python +red_nir_monthly_2017 = modis \ + .select( + col('granule_id'), + month('acquisition_date').alias('month'), + col('B01').alias('red'), + col('B02').alias('nir') + ) \ + .where( + (year('acquisition_date') == 2017) & + (dayofmonth('acquisition_date') == 15) & + (col('granule_id') == 'h21v09') + ) +red_nir_monthly_2017.printSchema() +``` + +### Step 3: Read tiles + +```python, step_3_python +red_nir_tiles_monthly_2017 = spark.read.raster( + catalog=red_nir_monthly_2017, + catalog_col_names=['red', 'nir'], + tile_dimensions=(256, 256) +) +``` + +### Step 4: Compute aggregates + +```python, step_4_python, results='raw' +result = red_nir_tiles_monthly_2017 \ + .where(st_intersects( + st_reproject(rf_geometry(col('red')), rf_crs(col('red')).crsProj4, rf_mk_crs('EPSG:4326')), + st_makePoint(lit(34.870605), lit(-4.729727))) + ) \ + .groupBy('month') \ + .agg(rf_agg_stats(rf_normalized_difference(col('nir'), col('red'))).alias('ndvi_stats')) \ + .orderBy(col('month')) \ + .select('month', 'ndvi_stats.*') +result.show() +``` + +## SQL + +For convenience, we're going to evaluate SQL from the Python environment. The SQL fragments should work in the `spark-sql` shell just the same. + +```python, sql_setup +def sql(stmt): + return spark.sql(stmt) +``` + +### Step 1: Load the catalog + +```python, step_1_sql, results='hidden' +sql("CREATE OR REPLACE TEMPORARY VIEW modis USING `aws-pds-modis-catalog`") +``` + +### Step 2: Down-select data by month + +```python, step_2_sql, results='raw' +sql(""" +CREATE OR REPLACE TEMPORARY VIEW red_nir_monthly_2017 AS +SELECT granule_id, month(acquisition_date) as month, B01 as red, B02 as nir +FROM modis +WHERE year(acquisition_date) = 2017 AND day(acquisition_date) = 15 AND granule_id = 'h21v09' +""") +sql('DESCRIBE red_nir_monthly_2017').show() +``` + +### Step 3: Read tiles + +```python, step_3_sql, results='hidden' +sql(""" +CREATE OR REPLACE TEMPORARY VIEW red_nir_tiles_monthly_2017 +USING raster +OPTIONS ( + catalogTable='red_nir_monthly_2017', + catalogColumns='red,nir', + tileDimensions='256,256' + ) +""") +``` + +### Step 4: Compute aggregates + +```python, step_4_sql, results='raw' +sql(""" +SELECT month, ndvi_stats.* FROM ( + SELECT month, rf_agg_stats(rf_normalized_difference(nir, red)) as ndvi_stats + FROM red_nir_tiles_monthly_2017 + WHERE st_intersects(st_reproject(rf_geometry(red), rf_crs(red), 'EPSG:4326'), st_makePoint(34.870605, -4.729727)) + GROUP BY month + ORDER BY month +) +""").show() +``` + +## Scala + +The latest Scala API documentation is available here: + +* [Scala API Documentation](https://rasterframes.io/latest/api/index.html) + + +### Step 1: Load the catalog + +```scala +import geotrellis.proj4.LatLng +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ +import org.apache.spark.sql._ +import org.apache.spark.sql.functions._ + + +implicit val spark = SparkSession.builder() + .master("local[*]") + .appName("RasterFrames") + .withKryoSerialization + .getOrCreate() + .withRasterFrames + +import spark.implicits._ + +val modis = spark.read.format("aws-pds-modis-catalog").load() +``` + +### Step 2: Down-select data by month + +```scala +val red_nir_monthly_2017 = modis + .select($"granule_id", month($"acquisition_date") as "month", $"B01" as "red", $"B02" as "nir") + .where(year($"acquisition_date") === 2017 && (dayofmonth($"acquisition_date") === 15) && $"granule_id" === "h21v09") +``` + +### Step 3: Read tiles + +```scala +val red_nir_tiles_monthly_2017 = spark.read.raster + .fromCatalog(red_nir_monthly_2017, "red", "nir") + .load() +``` + +### Step 4: Compute aggregates + +```scala +val result = red_nir_tiles_monthly_2017 + .where(st_intersects( + st_reproject(rf_geometry($"red"), rf_crs($"red"), LatLng), + st_makePoint(34.870605, -4.729727) + )) + .groupBy("month") + .agg(rf_agg_stats(rf_normalized_difference($"nir", $"red")) as "ndvi_stats") + .orderBy("month") + .select("month", "ndvi_stats.*") + +result.show() +``` diff --git a/pyrasterframes/src/main/python/docs/local-algebra.pymd b/pyrasterframes/src/main/python/docs/local-algebra.pymd new file mode 100644 index 000000000..8707196e3 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/local-algebra.pymd @@ -0,0 +1,70 @@ +# Local Map Algebra + +```python, setup, echo=False +from IPython.core.display import display +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import rf_normalized_difference, rf_tile, rf_dimensions, rf_extent, rf_tile_mean +import pyrasterframes.rf_ipython + +import pandas as pd +import os + +spark = create_rf_spark_session() +``` + +[Local map algebra](https://gisgeography.com/map-algebra-global-zonal-focal-local/) raster operations are element-wise operations on a single _tile_, between a _tile_ and a scalar, between two _tiles_, or among many _tiles_. These operations are common in processing of earth observation imagery and other image data. + +local op + +Credit: [GISGeography](https://gisgeography.com) + + +## Computing NDVI + +Here is an example of computing the Normalized Differential Vegetation Index (NDVI). NDVI is a vegetation index which emphasizes differences in relative biomass and vegetation health. The term _index_ in Earth observation means a combination of many raster bands into a single band that highlights a phenomenon of interest. Various indices have proven useful visual tools and frequently appear as features in machine learning models using Earth observation data. + +> NDVI is often used worldwide to monitor drought, monitor and predict agricultural production, assist in predicting hazardous fire zones, and map desert encroachment. The NDVI is preferred for global vegetation monitoring because it helps to compensate for changing illumination conditions, surface slope, aspect, and other extraneous factors (Lillesand. _Remote sensing and image interpretation_. 2004) + +We will apply the @ref:[catalog pattern](raster-catalogs.md) for defining the data we wish to process. To compute NDVI we need to compute local algebra on the *red* and *near infrared* (nir) bands: + + nir - red + NDVI = --------- + nir + red + +This form of `(x - y) / (x + y)` is common in remote sensing and is called a normalized difference. It is used with other band pairs to highlight water, snow, and other phenomena. + +```python, read_rasters +bands = {4: 'red', 8: 'nir'} +uri_pattern = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B0{}.tif' +catalog_df = pd.DataFrame([ + {bands[b_num]: uri_pattern.format(b_num) for b_num in bands.keys()} +]) +df = spark.read.raster(catalog=catalog_df.to_csv(index=None), + catalog_col_names=list(catalog_df.columns)) +df.printSchema() +``` + +Observe how the bands we need to use to compute the index are arranged as columns of our resulting DataFrame. The rows or observations are various spatial extents within the entire coverage area. + +RasterFrames provides a wide variety of local map algebra functions. There are several different broad categories, based on how many _tiles_ the function takes as input: + + * A function on a single _tile_ is a unary operation; example: @ref:[rf_log](reference.md#rf-log); + * A function on two _tiles_ is a binary operation; example: @ref:[rf_local_multiply](reference.md#rf-local-multiply); + * A function on a _tile_ and a scalar is a binary operation; example: @ref:[rf_local_less](reference.md#rf-local-less); or + * A function on many _tiles_ is a n-ary operation; example: @ref:[rf_agg_local_min](reference.md#rf-agg-local-min) + +We can express the normalized difference with a combination of `rf_local_divide`, `rf_local_subtract`, and `rf_local_add`. Since the normalized difference is so common, there is a convenience method `rf_normalized_difference`, which we use in this example. We will append a new column to the DataFrame, which will apply the map alegbra function to each row. + +```python, compute_ndvi +df = df.withColumn('ndvi', rf_normalized_difference(df.nir, df.red)) +df.printSchema() +``` + +We can inspect a sample of the data. Yellow indicates very healthy vegetation, and purple represents bare soil or impervious surfaces. + +```python, show_tile +t = df.select(rf_tile('ndvi').alias('ndvi')).first()['ndvi'] +display(t) +``` + +We continue examining NDVI in the @ref:[time series](time-series.md) section. diff --git a/pyrasterframes/src/main/python/docs/machine-learning.md b/pyrasterframes/src/main/python/docs/machine-learning.md new file mode 100644 index 000000000..aadebde8e --- /dev/null +++ b/pyrasterframes/src/main/python/docs/machine-learning.md @@ -0,0 +1,15 @@ +# Machine Learning + +RasterFrames provides facilities to train and predict with a wide variety of machine learning models through [Spark ML Pipelines](https://spark.apache.org/docs/latest/ml-guide.html). This library provides a variety of pipeline components for supervised learning, unsupervised learning, and data preparation that can be used to represent and repeatably conduct a variety of tasks in machine learning. + +The following sections provide some examples on how to integrate these workflows with RasterFrames. + +@@@ index + +* [Unsupervised Machine Learning](unsupervised-learning.md) +* [Supervised Machine Learning](supervised-learning.md) + +@@@ + + +@@toc { depth=2 } \ No newline at end of file diff --git a/pyrasterframes/src/main/python/docs/nodata-handling.pymd b/pyrasterframes/src/main/python/docs/nodata-handling.pymd new file mode 100644 index 000000000..568da4a75 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/nodata-handling.pymd @@ -0,0 +1,318 @@ +# "NoData" Handling + +## What is NoData? + +In raster operations, the preservation and correct processing of missing observations is very important. In [most DataFrames and scientific computing](https://www.oreilly.com/learning/handling-missing-data), the idea of missing data is expressed as a `null` or `NaN` value. However, a great deal of raster data is stored for space efficiency, which typically leads to use of integral values with a ["sentinel" value](https://en.wikipedia.org/wiki/Sentinel_value) designated to represent missing observations. This sentinel value varies across data products and is usually called the "NoData" value. + +RasterFrames provides a variety of functions to inspect and manage NoData within _tiles_. + +## Cell Types + +To understand how NoData is handled in RasterFrames, we first need to understand the different underlying types of data called cell types. RasterFrames cell types are GeoTrellis `CellType`s, so the [GeoTrellis documentation](https://geotrellis.readthedocs.io/en/latest/guide/core-concepts.html?#working-with-cell-values) is a valuable resource on how these are defined. + +```python setup, echo=False +import pyrasterframes +from pyrasterframes.rasterfunctions import * +import pyrasterframes.rf_ipython +from IPython.display import display +import pandas as pd +import numpy as np +from pyrasterframes.rf_types import Tile + +spark = pyrasterframes.get_spark_session() +``` + +The `CellType` class from the `rf_types` submodule allows us to create a representation of any valid cell type. There are convenience methods to create instances for a variety of basic types. + +```python, celltype_ctors, results='hidden' +from pyrasterframes.rf_types import CellType +CellType.bool() +CellType.int8() +CellType.uint8() +CellType.int16() +CellType.uint16() +CellType.int32() +CellType.float32() +CellType.float64() +``` + +We can also inspect the cell type of a given _tile_ or `proj_raster` column. + +```python, ct_from_sen, results='raw' +spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif') \ + .select(rf_cell_type('proj_raster')).distinct().show() +``` + +### Understanding Cell Types and NoData + +We can use the methods on the `CellType` class to learn more about a specific cell type. Let's consider the cell type of our sample data above. + +```python, raw_example +ct = CellType('uint16raw') +ct, ct.is_floating_point(), ct.has_no_data() +``` + +We can see that for the above data source, there is no defined NoData value. This means that each value is interpreted as a valid observation. Often such data is meant to be combined with another band indicating the quality of observations at each location. The lack of NoData is indicated by the `raw` at the end of the type name. Consider for contrast the `uint16` cell type. + +```python, non_raw_example +from pyrasterframes.rf_types import CellType +ct = CellType('uint16') +ct, ct.is_floating_point(), ct.has_no_data(), ct.no_data_value() +``` + +In this case, the minimum value of 0 is designated as the NoData value. For integral-valued cell types, the NoData is typically zero, the maximum, or the minimum value for the underlying data type. The NoData value can also be a user-defined value. In that case the value is designated with `ud`. + +```python, cell_type_name +CellType.uint16().with_no_data_value(99).cell_type_name +``` + +Floating point types have `NaN` as the NoData value by default. However, a user-defined NoData can be set. + +```python, float_ud +print(CellType.float32().no_data_value()) +print(CellType.float32().with_no_data_value(-99.9).no_data_value()) +``` + +## Masking + +Let's continue the example above with Sentinel-2 data. Band 2 is blue and has no defined NoData. The quality information is in a separate file called the scene classification (SCL), which delineates areas of missing data and probable clouds. For more information on this, see the [Sentinel-2 algorithm overview](https://earth.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm). Figure 3 tells us how to interpret the scene classification. For this example, we will exclude NoData, defective pixels, probable clouds, and cirrus clouds: values 0, 1, 8, 9, and 10. + +![Sentinel-2 Scene Classification Values](static/sentinel-2-scene-classification-labels.png) + +Credit: [Sentinel-2 algorithm overview](https://earth.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm) + +The first step is to create a catalog with our band of interest and the SCL band. We read the data from the catalog, so the blue band and SCL _tiles_ are aligned across rows. + +```python, blue_scl_cat +from pyspark.sql import Row + +blue_uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif' +scl_uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/SCL.tif' +cat = spark.createDataFrame([Row(blue=blue_uri, scl=scl_uri),]) +unmasked = spark.read.raster(catalog=cat, catalog_col_names=['blue', 'scl']) +unmasked.printSchema() +``` + +```python, show_cell_types, results='raw' +unmasked.select(rf_cell_type('blue'), rf_cell_type('scl')).distinct().show() +``` + +Drawing on @ref:[local map algebra](local-algebra.md) techniques, we will create new _tile_ columns that are indicators of unwanted pixels, as defined above. Since the mask column is an integer type, the addition is equivalent to a logical or, so the boolean true values are 1. + +```python, def_mask, results='raw' +from pyspark.sql.functions import lit + +mask_part = unmasked.withColumn('nodata', rf_local_equal('scl', lit(0))) \ + .withColumn('defect', rf_local_equal('scl', lit(1))) \ + .withColumn('cloud8', rf_local_equal('scl', lit(8))) \ + .withColumn('cloud9', rf_local_equal('scl', lit(9))) \ + .withColumn('cirrus', rf_local_equal('scl', lit(10))) + +one_mask = mask_part.withColumn('mask', rf_local_add('nodata', 'defect')) \ + .withColumn('mask', rf_local_add('mask', 'cloud8')) \ + .withColumn('mask', rf_local_add('mask', 'cloud9')) \ + .withColumn('mask', rf_local_add('mask', 'cirrus')) + +one_mask.select(rf_cell_type('mask')).distinct().show() +``` + +Because there is not a NoData already defined, we will choose one. In this particular example, the minimum value is greater than zero, so we can use 0 as the NoData value. + +```python, pick_nd, results='raw' +one_mask.agg(rf_agg_stats('blue').min.alias('blue_min')).show() +``` + +We can now construct the cell type string for our blue band's cell type, designating 0 as NoData. + +```python, get_ct_string +blue_ct = one_mask.select(rf_cell_type('blue')).distinct().first()[0][0] +masked_blue_ct = CellType(blue_ct).with_no_data_value(0) +masked_blue_ct.cell_type_name +``` + +Now we will use the @ref:[`rf_mask_by_value`](reference.md#rf-mask-by-value) to designate the cloudy and other unwanted pixels as NoData in the blue column by converting the cell type and applying the mask. + +```python, mask_blu +with_nd = rf_convert_cell_type('blue', masked_blue_ct) +masked = one_mask.withColumn('blue_masked', + rf_mask_by_value(with_nd, 'mask', lit(1))) \ + .drop('nodata', 'defect', 'cloud8', 'cloud9', 'cirrus', 'blue') +``` + +We can verify that the number of NoData cells in the resulting `blue_masked` column matches the total of the boolean `mask` _tile_ to ensure our logic is correct. + +```python, show_masked, results='raw' +masked.select(rf_no_data_cells('blue_masked'), rf_tile_sum('mask')).show(10) +``` + +It's also nice to view a sample. The white regions are areas of NoData. + +```python, display_blu, caption='Blue band masked against selected SCL values' +sample = masked.orderBy(-rf_no_data_cells('blue_masked')).select(rf_tile('blue_masked'), rf_tile('scl')).first() +display(sample[0]) +``` + +And the original SCL data. The bright yellow is a cloudy region in the original image. + +```python, display_scl, caption='SCL tile for above' +display(sample[1]) +``` + +## NoData and Local Arithmetic + +Let's now explore how the presence of NoData affects @ref:[local map algebra](local-algebra.md) operations. To demonstrate the behavior, lets create two _tiles_. One _tile_ will have values of 0 and 1, and the other will have values of just 0. + +```python, local_arith +tile_size = 100 +x = np.zeros((tile_size, tile_size), dtype='int16') +x[:,tile_size//2:] = 1 +x = Tile(x) +y = Tile(np.zeros((tile_size, tile_size), dtype='int16')) + +rf = spark.createDataFrame([Row(x=x, y=y)]) +print('x') +display(x) +``` + +```python, display_1 +print('y') +display(y) +``` + +Now, let's create a new column from `x` with the value of 1 changed to NoData. Then, we will add this new column with NoData to the `y` column. As shown below, the result of the sum also has NoData (represented in white). In general for local algebra operations, Data + NoData = NoData. + +```python, display_2 +masked_rf = rf.withColumn('x_nd', rf_mask_by_value('x', 'x', lit(1)) ) +masked_rf = masked_rf.withColumn('x_nd_y_sum', rf_local_add('x_nd', 'y')) +row = masked_rf.collect()[0] +print('x with NoData') +display(row.x_nd) +``` + +```python, display_3 +print('x with NoData plus y') +display(row.x_nd_y_sum) +``` +To see more information about possible operations on _tile_ columns, see the @ref:[local map algebra](local-algebra.md) page and @ref:[function reference](reference.md#local-map-algebra). + +## Changing a Tile's NoData Values + +One way to mask a _tile_ is to make a new _tile_ with a user defined NoData value. We will explore this method below. First, lets create a DataFrame from a _tile_ with values of 0, 1, 2, and 3. We will use numpy to create a 100x100 _tile_ with vertical bands containing values 0, 1, 2, and 3. + +```python, create_dummy_tile, caption='Dummy Tile' +tile_size = 100 +x = np.zeros((tile_size, tile_size), dtype='int16') + +# setting the values of the columns +for i in range(4): + x[:, i*tile_size//4:(i+1)*tile_size//4] = i +x = Tile(x) + +rf = spark.createDataFrame([Row(tile=x)]) +display(x) +``` + +First, we mask the value of 1 by making a new _tile_ column with the user defined cell type 'uint16ud1'. Then, we mask out the value of two by making a _tile_ column with the cell type 'uint16ud2'. + +```python, masked_rf +def get_nodata_ct(nd_val): + return CellType('uint16').with_no_data_value(nd_val) + +masked_rf = rf.withColumn('tile_nd_1', + rf_convert_cell_type('tile', get_nodata_ct(1))) \ + .withColumn('tile_nd_2', + rf_convert_cell_type('tile_nd_1', get_nodata_ct(2))) \ +``` + +```python, masked_collected +collected = masked_rf.collect() +``` + +Let's look at the new _tiles_ we created. The _tile_ named `tile_nd_1` has the 1 values masked out as expected. + +```python, display_4 +display(collected[0].tile_nd_1) +``` + +And the _tile_ named `tile_nd_2` has the values of 1 and 2 masked out. This is because we created the _tile_ by setting a new user defined NoData value to `tile_nd_1`, and the values previously masked out in `tile_nd_1` stayed masked when creating `tile_nd_2`. + +```python, display_5 +display(collected[0].tile_nd_2) +``` + + +## Combining Tiles with Different Data Types + +RasterFrames supports having _tile_ columns with different cell types in a single DataFrame. It is important to understand how these different cell types interact. + +Let's first create a RasterFrame that has columns of `float` and `int` cell type. + +```python, show_1, results='raw' +x = Tile((np.ones((100, 100))*2), CellType.float64()) +y = Tile((np.ones((100, 100))*3), CellType.int32()) +rf = spark.createDataFrame([Row(x=x, y=y)]) + +rf.select(rf_cell_type('x'), rf_cell_type('y')).distinct().show() +``` + +When performing a local operation between _tile_ columns with cell types `int` and `float`, the resulting _tile_ cell type will be `float`. In local algebra over two _tiles_ of different "sized" cell types, the resulting cell type will be the larger of the two input _tiles'_ cell types. + +```python, show_2, results='raw' +rf.select( + rf_cell_type('x'), + rf_cell_type('y'), + rf_cell_type(rf_local_add('x', 'y')).alias('xy_sum'), + ).show(1) +``` + +Combining _tile_ columns of different cell types gets a little trickier when user defined NoData cell types are involved. Let's create two _tile_ columns: one with a NoData value of 1, and one with a NoData value of 2 (using our previously defined `get_nodata_ct` function). + +```python, create_ones +x_nd_1 = Tile((np.ones((100, 100))*3), get_nodata_ct(1)) +x_nd_2 = Tile((np.ones((100, 100))*3), get_nodata_ct(2)) +rf_nd = spark.createDataFrame([Row(x_nd_1=x_nd_1, x_nd_2=x_nd_2)]) +``` + +Let's try adding the _tile_ columns with different NoData values. When there is an inconsistent NoData value in the two columns, the NoData value of the right-hand side of the sum is kept. In this case, this means the result has a NoData value of 1. + +```python, show_3, results='raw' +rf_nd_sum = rf_nd.withColumn('x_nd_sum', rf_local_add('x_nd_2', 'x_nd_1')) +rf_nd_sum.select(rf_cell_type('x_nd_sum')).distinct().show() +``` + +Reversing the order of the sum changes the NoData value of the resulting column to 2. + +```python, show_4, results='raw' +rf_nd_sum = rf_nd.withColumn('x_nd_sum', rf_local_add('x_nd_1', 'x_nd_2')) +rf_nd_sum.select(rf_cell_type('x_nd_sum')).distinct().show() +``` + +## NoData Values in Aggregation + +Let's use the same _tile_ as before to demonstrate how NoData values affect _tile_ aggregations. + +```python, display_6 +tile_size = 100 +x = np.zeros((tile_size, tile_size)) +for i in range(4): + x[:, i*tile_size//4:(i+1)*tile_size//4] = i +x = Tile(x, CellType.int16()) + +rf = spark.createDataFrame([Row(tile=x)]) +display(x) +``` + +First we create the two new masked _tile_ columns as before. One with only the value of 1 masked, and the other with and values of 1 and 2 masked. + +```python, cell_type_2 +masked_rf = rf.withColumn('tile_nd_1', + rf_convert_cell_type('tile', get_nodata_ct(1))) \ + .withColumn('tile_nd_2', + rf_convert_cell_type('tile_nd_1', get_nodata_ct(2))) +``` + +The results of `rf_tile_sum` vary on the _tiles_ that were masked. This is because any cells with NoData values are ignored in the aggregation. Note that `tile_nd_2` has the lowest sum, since it has the fewest amount of data cells. + +```python, show_5, results='raw' +masked_rf.select(rf_tile_sum('tile'), rf_tile_sum('tile_nd_1'), rf_tile_sum('tile_nd_2')).show() +``` diff --git a/pyrasterframes/src/main/python/docs/numpy-pandas.pymd b/pyrasterframes/src/main/python/docs/numpy-pandas.pymd new file mode 100644 index 000000000..537ffe3df --- /dev/null +++ b/pyrasterframes/src/main/python/docs/numpy-pandas.pymd @@ -0,0 +1,119 @@ +# NumPy and Pandas + +In the Python Spark API, the work of distributed computing over the DataFrame is done on many executors (the Spark term for workers) inside Java virtual machines (JVM). Most calls to `pyspark` are passed to a Java process via the `py4j` library. The user can also ask for data inside the JVM to be brought over to the Python driver (the Spark term for the client application). When dealing with _tiles_, the driver will receive this data as a lightweight wrapper object around a NumPy ndarray. It is also possible to write lambda functions against NumPy arrays and evaluate them in the Spark DataFrame. + +## Performance Considerations + +When working with large, distributed datasets in Spark, attention is required when invoking _actions_ on the data. In general, _transformations_ are lazily evaluated in Spark, meaning the code runs fast and it doesn't move any data around. But _actions_ cause the evaluation to happen, meaning all the lazily planned _transformations_ are going to be computed and data is going to be processed and moved around. In general, if a [`pyspark` function](https://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html) returns a DataFrame, it is probably a _transformation_, and if not, it is an _action_. + +When many _actions_ are invoked, a lot of data can flow from executors to the driver. In `pyspark`, the data then has to move from the driver JVM to the Python process running the driver. When that happens, if there are any _tiles_ in the data, they will be converted to a Python [`Tile`](https://github.com/locationtech/rasterframes/blob/develop/pyrasterframes/src/main/python/pyrasterframes/rf_types.py) object. In practical work with Earth observation data, the _tiles_ are frequently 256 by 256 arrays, which may be 100kb or more each. Individually they are small, but a DataFrame can easily have dozens of such _tile_ columns and millions of rows. + +All of this discussion reinforces two important principles for working with Spark: understanding the cost of an _action_ and using @ref:[aggreates](aggregation.md), summaries, or samples to manage the cost of _actions_. + +## The `Tile` Class + +In Python, _tiles_ are represented with the `rf_types.Tile` class. This is a NumPy `ndarray` with two dimensions, along with some additional metadata allowing correct conversion to the GeoTrellis @ref:[cell type](nodata-handling.md#cell-types). + +```python tile_intro +from pyrasterframes.rf_types import Tile +import numpy as np + +t = Tile(np.random.randn(4, 4)) +print(str(t)) +``` + +You can access the NumPy array with the `cells` member of `Tile`. + +```python tile_cells +t.cells.shape, t.cells.nbytes +``` + +## DataFrame `toPandas` + +As discussed in the @ref:[raster writing chapter](raster-write.md#dataframe-samples), a pretty display of Pandas DataFrame containing _tiles_ is available by importing the `rf_ipython` submodule. In addition, as discussed in the @ref:[vector data chapter](vector-data.md), any geometry type in the Spark DataFrame will be converted into a Shapely geometry. Taken together, we can easily get the spatial information and raster data as a NumPy array, all within a Pandas DataFrame. + +```python spark_session, echo=False +import pyrasterframes +from pyrasterframes.rasterfunctions import * +from IPython.display import display +spark = pyrasterframes.get_spark_session() + +``` + +```python toPandas +import pyrasterframes.rf_ipython +from pyspark.sql.functions import lit, col + +cat = spark.read.format('aws-pds-modis-catalog').load() \ + .filter( + (col('granule_id') == 'h11v04') & + (col('acquisition_date') > lit('2018-02-19')) & + (col('acquisition_date') < lit('2018-02-22')) + ) + +spark_df = spark.read.raster(catalog=cat, catalog_col_names=['B01']) \ + .select( + 'acquisition_date', + 'granule_id', + rf_tile('B01').alias('tile'), + rf_geometry('B01').alias('tile_geom') + ) + +pandas_df = spark_df.limit(10).toPandas() +pandas_df.iloc[0].apply(lambda v: type(v)) +``` + +## User Defined Functions + +As we demonstrated with @ref:[vector data](vector-data.md#shapely-geometry-support), we can also make use of the `Tile` type to create [user-defined functions (UDF)](https://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.functions.udf) that can take a _tile_ as input, return a _tile_ as output, or both. Here is a trivial and **inefficient** example of doing both. A serious performance implication of user defined functions in Python is that all the executors must move the Java objects to Python, evaluate the function, and then move the Python objects back to Java. Use the many @ref:[built-in functions](reference.md) wherever possible, and ask the [community](https://gitter.im/locationtech/rasterframes) if you have an idea for a function that should be included. + +We will demonstrate an example of creating a UDF that is logically equivalent to a built-in function. We'll quickly show that the resulting _tiles_ are approximately equivalent. The reason they are not exactly the same is that one is computed in Python and the other is computed in Java. + +```python udf +from pyrasterframes.rf_types import TileUDT +from pyspark.sql.functions import udf + +@udf(TileUDT()) +def my_udf(t): + import numpy as np + return Tile(np.log1p(t.cells)) + +udf_df = spark_df.limit(1).select( + my_udf('tile').alias('udf_result'), + rf_log1p('tile').alias('built_in_result') + ).toPandas() + +row = udf_df.iloc[0] +diff = row['udf_result'] - row['built_in_result'] +print(type(diff)) +np.abs(diff.cells).max() +``` + +We can also inspect an image of the difference between the two _tiles_, which is just random noise. Both _tiles_ have the same structure of NoData, as exhibited by the white areas. + +```python udf_diff_noise_tile +display(diff) +``` + +## Creating a Spark DataFrame + +You can also create a Spark DataFrame with a column full of `Tile` objects or Shapely geomtery objects. + +The example below will create a Pandas DataFrame with ten rows of noise _tiles_ and random `Point`s. We will then create a Spark DataFrame from it. + +```python create_spark_df +import pandas as pd +from shapely.geometry import Point + +pandas_df = pd.DataFrame([ + { + 'tile': Tile(np.random.randn(100, 100)), + 'geom': Point(-90 + 90 * np.random.random((2, 1))) + } for _ in range(10) + ]) + +spark_df = spark.createDataFrame(pandas_df) + +spark_df.printSchema() +spark_df.count() +``` diff --git a/pyrasterframes/src/main/python/docs/raster-catalogs.pymd b/pyrasterframes/src/main/python/docs/raster-catalogs.pymd new file mode 100644 index 000000000..15baddab9 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/raster-catalogs.pymd @@ -0,0 +1,195 @@ +# Raster Catalogs + +While interesting processing can be done on a @ref:[single raster file](raster-read.md#single-raster), RasterFrames shines when _catalogs_ of raster data are to be processed. In its simplest form, a _catalog_ is a list of @ref:[URLs referencing raster files](raster-read.md#uri-formats). This list can be a Spark DataFrame, Pandas DataFrame, CSV file or CSV string. The _catalog_ is input into the `raster` DataSource described in the @ref:[next page](raster-read.md), which creates _tiles_ from the rasters at the referenced URLs. + +A _catalog_ can have one or two dimensions: + +* One-D: A single column contains raster URLs across the rows. All referenced rasters represent the same @ref:[band](concepts.md#band). For example, a column of URLs to Landsat 8 near-infrared rasters covering Europe. Each row represents different places and times. +* Two-D: Many columns contain raster URLs. Each column references the same band, and each row represents the same place and time. For example, red-, green-, and blue-band columns for scenes covering Europe. Each row represents a single @ref:[scene](concepts.md#scene) with the same resolution, extent, [_CRS_][CRS], etc across the row. + +## Creating a Catalog + +This section will provide some examples of _catalogs_ creation, as well as introduce some experimental _catalogs_ built into RasterFrames. Reading raster data represented by a _catalog_ is covered in more detail in the @ref:[next page](raster-read.md). + +```python, setup, echo=False +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import * +import pyrasterframes.rf_ipython +import pandas as pd +spark = create_rf_spark_session() +``` + +### One-D + +A single URL is the simplest form of a catalog. + +```python, oned_onerow_catalog +file_uri = "/data/raster/myfile.tif" +# Pandas DF +my_cat = pd.DataFrame({'B01': [file_uri]}) + +# equivalent Spark DF +from pyspark.sql import Row +my_cat = spark.createDataFrame([Row(B01=file_uri)]) + +#equivalent CSV string +my_cat = "B01\n{}".format(file_uri) +``` + +A single column represents the same content type with different observations along the rows. In this example it is band 1 of MODIS surface reflectance, which is visible red. In the example the location of the images is the same, indicated by the granule identifier `h04v09`, but the dates differ: 2018185 (July 4, 2018) and 2018188 (July 7, 2018). + +```python, oned_tworow_catalog +scene1_B01 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018185/MCD43A4.A2018185.h04v09.006.2018194032851_B01.TIF" +scene2_B01 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018188/MCD43A4.A2018188.h04v09.006.2018198232008_B01.TIF" + +# a pandas DF +one_d_cat_pd = pd.DataFrame({'B01': [scene1_B01, scene2_B01]}) + +# equivalent spark DF +one_d_cat_df = spark.createDataFrame([Row(B01=scene1_B01), Row(B01=scene2_B01)]) + +# equivalent CSV string +one_d_cat_csv = '\n'.join(['B01', scene1_B01, scene2_B01]) +``` + +This is what it looks like in DataFrame form: + +```python, view_spark_cat_1 +one_d_cat_df.show(truncate=False) +``` + +### Two-D + +In this example, multiple columns representing multiple content types (bands) across multiple scenes. In each row, the scene is the same: granule id `h04v09` on July 4 or July 7, 2018. The first column is band 1, red, and the second is band 2, near infrared. + +```python, twod_catalog +scene1_B01 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018185/MCD43A4.A2018185.h04v09.006.2018194032851_B01.TIF" +scene1_B02 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018185/MCD43A4.A2018185.h04v09.006.2018194032851_B02.TIF" +scene2_B01 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018188/MCD43A4.A2018188.h04v09.006.2018198232008_B01.TIF" +scene2_B02 = "https://modis-pds.s3.amazonaws.com/MCD43A4.006/04/09/2018188/MCD43A4.A2018188.h04v09.006.2018198232008_B02.TIF" + +# Pandas DF +two_d_cat_pd = pd.DataFrame([ + {'B01': [scene1_B01], 'B02': [scene1_B02]}, + {'B01': [scene2_B01], 'B02': [scene2_B02]} +]) + +# or +two_d_cat_df = spark.createDataFrame([ + Row(B01=scene1_B01, B02=scene1_B02), + Row(B01=scene2_B01, B02=scene2_B02) +]) + +# As CSV string +tow_d_cat_csv = '\n'.join(['B01,B02', scene1_B01 + "," + scene1_B02, scene2_B01 + "," + scene2_B02]) +``` + +This is what it looks like in DataFrame form: + +```python, view_spark_cat_1 +two_d_cat_df.show(truncate=False) +``` + +## Using External Catalogs + +The concept of a _catalog_ is much more powerful when we consider examples beyond constructing the DataFrame, and instead read the data from an external source. Here's an extended example of reading a cloud-hosted CSV file containing MODIS scene metadata and transforming it into a _catalog_. The metadata describing the content of each URL is an important aspect of processing raster data. + +```python, remote_csv, results='raw' +from pyspark import SparkFiles +from pyspark.sql import functions as F + +spark.sparkContext.addFile("https://modis-pds.s3.amazonaws.com/MCD43A4.006/2018-07-04_scenes.txt") + +scene_list = spark.read \ + .format("csv") \ + .option("header", "true") \ + .load(SparkFiles.get("2018-07-04_scenes.txt")) +scene_list.show(4, truncate=False) +``` + +Observe the scenes list file has URIs to `index.html` files in the download_url column. The image URI's are in the same directory. The filenames are of the form `${gid}_B${band}.TIF`. The next code chunk builds these URIs, which completes our catalog. + +```python, show_remote_catalog, results='raw' +modis_catalog = scene_list \ + .withColumn('base_url', + F.concat(F.regexp_replace('download_url', 'index.html$', ''), 'gid',) + ) \ + .withColumn('B01' , F.concat('base_url', F.lit("_B01.TIF"))) \ + .withColumn('B02' , F.concat('base_url', F.lit("_B02.TIF"))) \ + .withColumn('B03' , F.concat('base_url', F.lit("_B03.TIF"))) +modis_catalog.show(4, truncate=True) +``` + +## Using Built-in Catalogs + +RasterFrames comes with two experimental catalogs over the AWS PDS [Landsat 8][Landsat] and [MODIS][MODIS] repositories. They are created by downloading the latest scene lists and building up the appropriate band URI columns as in the prior example. + +> Note: The first time you run these may take some time, as the catalogs are large and have to be downloaded. However, they are cached and subsequent invocations should be faster. + +### MODIS + +```python, evaluate=False +modis_catalog = spark.read.format('aws-pds-modis-catalog').load() +modis_catalog.printSchema() +``` +``` +root + |-- product_id: string (nullable = false) + |-- acquisition_date: timestamp (nullable = false) + |-- granule_id: string (nullable = false) + |-- gid: string (nullable = false) + |-- B01: string (nullable = true) + |-- B01qa: string (nullable = true) + |-- B02: string (nullable = true) + |-- B02qa: string (nullable = true) + |-- B03: string (nullable = true) + |-- B03aq: string (nullable = true) + |-- B04: string (nullable = true) + |-- B04qa: string (nullable = true) + |-- B05: string (nullable = true) + |-- B05qa: string (nullable = true) + |-- B06: string (nullable = true) + |-- B06qa: string (nullable = true) + |-- B07: string (nullable = true) + |-- B07qa: string (nullable = true) +``` + +### Landsat 8 + +The Landsat 8 catalog includes a richer set of metadata describing the contents of each scene. + +```python, evaluate=False +l8 = spark.read.format('aws-pds-l8-catalog').load() +l8.printSchema() +``` +``` +root + |-- product_id: string (nullable = false) + |-- entity_id: string (nullable = false) + |-- acquisition_date: timestamp (nullable = false) + |-- cloud_cover_pct: float (nullable = false) + |-- processing_level: string (nullable = false) + |-- path: short (nullable = false) + |-- row: short (nullable = false) + |-- bounds_wgs84: struct (nullable = false) + | |-- minX: double (nullable = false) + | |-- maxX: double (nullable = false) + | |-- minY: double (nullable = false) + | |-- maxY: double (nullable = false) + |-- B1: string (nullable = true) + |-- B2: string (nullable = true) + |-- B3: string (nullable = true) + |-- B4: string (nullable = true) + |-- B5: string (nullable = true) + |-- B6: string (nullable = true) + |-- B7: string (nullable = true) + |-- B8: string (nullable = true) + |-- B9: string (nullable = true) + |-- B10: string (nullable = true) + |-- B11: string (nullable = true) + |-- BQA: string (nullable = true) +``` + +[MODIS]: https://docs.opendata.aws/modis-pds/readme.html +[Landsat]: https://docs.opendata.aws/landsat-pds/readme.html +[CRS]: https://en.wikipedia.org/wiki/Spatial_reference_system diff --git a/pyrasterframes/src/main/python/docs/raster-io.md b/pyrasterframes/src/main/python/docs/raster-io.md new file mode 100644 index 000000000..68696397c --- /dev/null +++ b/pyrasterframes/src/main/python/docs/raster-io.md @@ -0,0 +1,34 @@ +# Raster Data I/O + +The standard mechanism by which any data is brought in and out of a Spark Dataframe is the [Spark SQL DataSource][DS]. RasterFrames provides specialized DataSources for geospatial raster data and maintains compatibility with existing general purpose DataSources, such as Parquet. + +* @ref:[Catalog Readers](raster-catalogs.md) + - `aws-pds-l8-catalog`: built-in catalog over [Landsat on AWS][Landsat] + - `aws-pds-modis-catalog`: built-in catalog over [MODIS on AWS][MODIS] + - `geotrellis-catalog`: for enumerating [GeoTrellis layers][GTLayer] +* @ref:[Raster Readers](raster-read.md) + - `raster`: the standard reader for most raster data, including single raster files or catalogs + - `geotiff`: a simplified reader for reading a single GeoTIFF file + - `geotrellis`: for reading a [GeoTrellis layer][GTLayer] +* @ref:[Raster Writers](raster-write.md) + - @ref:[`geotiff`](raster-write.md#geotiffs): beta writer to GeoTiff file format + - @ref:[`geotrellis`](raster-write.md#geotrellis-layers): creating a [GeoTrellis layer][GTLayer] + - @ref:[`parquet`](raster-write.md#parquet): general purpose writer for [Parquet][Parquet] + + +Furthermore, when in a Jupyter Notebook environment, you can view @ref:[Tile](raster-write.md#tile-samples) and @ref:[DataFrame](raster-write.md#dataframe-samples) samples. + +There is also support for @ref:[vector data](vector-data.md) for masking and data labeling. + +@@@ index +* @ref:[Raster Catalogs](raster-catalogs.md) +* @ref:[Raster Readers](raster-read.md) +* @ref:[Raster Writers](raster-write.md) +@@@ + + +[DS]: https://spark.apache.org/docs/latest/sql-data-sources.html +[GTLayer]: https://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html +[Parquet]: https://spark.apache.org/docs/latest/sql-data-sources-parquet.html +[MODIS]: https://docs.opendata.aws/modis-pds/readme.html +[Landsat]: https://docs.opendata.aws/landsat-pds/readme.html diff --git a/pyrasterframes/src/main/python/docs/raster-processing.md b/pyrasterframes/src/main/python/docs/raster-processing.md new file mode 100644 index 000000000..a16b2bb7d --- /dev/null +++ b/pyrasterframes/src/main/python/docs/raster-processing.md @@ -0,0 +1,16 @@ +# Raster Processing + +@@@ index + +* @ref:[Local Algebra](local-algebra.md) +* @ref:["NoData" Handling](nodata-handling.md) +* @ref:[Aggregation](aggregation.md) +* @ref:[Time Series](time-series.md) +* @ref:[Machine Learning](machine-learning.md) + +@@@ + + +Explore what you can do with RasterFrames to work with, analyze, manipulate, and summarize Earth observation data. + +@@toc { depth=1 } \ No newline at end of file diff --git a/pyrasterframes/src/main/python/docs/raster-read.pymd b/pyrasterframes/src/main/python/docs/raster-read.pymd new file mode 100644 index 000000000..3a505993d --- /dev/null +++ b/pyrasterframes/src/main/python/docs/raster-read.pymd @@ -0,0 +1,224 @@ +# Reading Raster Data + +```python, setup, echo=False +from IPython.display import display +import pyrasterframes.rf_ipython +import pandas as pd +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import * +spark = create_rf_spark_session() +``` + +RasterFrames registers a DataSource named `raster` that enables reading of GeoTIFFs (and other formats when @ref:[GDAL is installed](getting-started.md#installing-gdal)) from arbitrary URIs. The `raster` DataSource operates on either a single raster file location or another DataFrame, called a _catalog_, containing pointers to many raster file locations. + +RasterFrames can also read from @ref:[GeoTrellis catalogs and layers](raster-read.md#geotrellis). + +## Single Raster + +The simplest way to use the `raster` reader is with a single raster from a single URI or file. In the examples that follow we'll be reading from a Sentinel-2 scene stored in an AWS S3 bucket. + +```python, read_one_uri +rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif') +rf.printSchema() +``` + +The file at the address above is a valid [Cloud Optimized GeoTIFF (COG)](https://www.cogeo.org/), which RasterFrames fully supports. RasterFrames will take advantage of the optimizations in the COG format to enable more efficient reading compared to non-COG GeoTIFFs. + +Let's unpack the `proj_raster` column and look at the contents in more detail. It contains a @ref:[_CRS_][CRS], a spatial @ref:[_extent_][Extent] measured in that CRS, and a two-dimensional array of numeric values called a @ref:[_tile_][Tile]. + +```python, select_crs +crs = rf.select(rf_crs("proj_raster").alias("value")).first() +print("CRS", crs.value.crsProj4) +``` + +```python, raster_parts, results='raw' +parts = rf.select( + rf_extent("proj_raster").alias("extent"), + rf_tile("proj_raster").alias("tile") +) +parts.show(5, truncate=False) +``` + +You can also see that the single raster has been broken out into many arbitrary non-overlapping regions. Doing so takes advantage of parallel in-memory reads from the cloud hosted data source and allows Spark to work on manageable amounts of data per task. The following code fragment shows us how many subtiles were created from a single source image. + +```python, count_by_uri, results='raw' +rf.groupby(rf.proj_raster_path).count().show(truncate=False) +``` + +Let's select a single _tile_ and view it. The _tile_ preview image as well as the string representation provide some basic information about the _tile_: its dimensions as numbers of columns and rows and the cell type, or data type of all the cells in the _tile_. For more about cell types, refer to @ref:[this discussion](nodata-handling.md#cell-types). + +```python, show_tile_sample +tile = rf.select(rf_tile("proj_raster")).first()[0] +display(tile) +``` + +## URI Formats + +RasterFrames relies on three different I/O drivers, selected based on a combination of scheme, file extentions, and library availability. GDAL is used by default if a compatible version of GDAL (>= 2.4) is installed, and if GDAL supports the specified scheme. If GDAL is not available, either the _Java I/O_ or _Hadoop_ driver will be selected, depending on scheme. + +Note: The GDAL driver is the only one that can read non-GeoTIFF files. + + +| Prefix | GDAL | Java I/O | Hadoop | +| ------------------- | ----------- | -------- | -------- | +| `gdal:////` | yes | no | no | +| `file://` | yes | yes | no | +| `http://` | yes | yes | no | +| `https://` | yes | yes | no | +| `ftp://` | `/vsicurl/` | yes | no | +| `hdfs://` | `/vsihdfs/` | no | yes | +| `s3://` | `/vsis3/` | yes | no | +| `s3n://` | no | no | yes | +| `s3a://` | no | no | yes | +| `wasb://` | `/vsiaz/` | no | yes | +| `wasbs://` | no | no | yes | + +Specific [GDAL Virtual File System drivers](https://gdal.org/user/virtual_file_systems.html) can be selected using the `gdal:////` syntax. For example If you have a `archive.zip` file containing a GeoTiff named `my-file-inside.tif`, you can address it with `gdal://vsizip//path/to/archive.zip/my-file-inside.tif`. Another example would be a MRF file in an S3 bucket on AWS: `gdal://vsis3/my-bucket/prefix/to/raster.mrf`. See the GDAL documentation for the format of the URIs after the `gdal:/` scheme. + + +## Raster Catalogs + +Consider the definition of a @ref:[_Catalog_](raster-catalogs.md) previously discussed, let's read the raster data contained in the catalog URIs. We will start with the @ref:[external catalog](raster-catalogs.md#using-external-catalogs) of MODIS surface reflectance. + +```python, catalog_prep +from pyspark import SparkFiles +from pyspark.sql import functions as F + +cat_filename = "2018-07-04_scenes.txt" +spark.sparkContext.addFile("https://modis-pds.s3.amazonaws.com/MCD43A4.006/{}".format(cat_filename)) + +modis_catalog = spark.read \ + .format("csv") \ + .option("header", "true") \ + .load(SparkFiles.get(cat_filename)) \ + .withColumn('base_url', + F.concat(F.regexp_replace('download_url', 'index.html$', ''), 'gid',) + ) \ + .drop('download_url') \ + .withColumn('red' , F.concat('base_url', F.lit("_B01.TIF"))) \ + .withColumn('nir' , F.concat('base_url', F.lit("_B02.TIF"))) + +modis_catalog.printSchema() + +print("Available scenes: ", modis_catalog.count()) +``` + +```python, show_catalog, results='raw' +modis_catalog.show(5, truncate=True) +``` + +MODIS data products are delivered on a regular, consistent grid, making identification of a specific area over time easy using [`(h,v)`](https://modis-land.gsfc.nasa.gov/MODLAND_grid.html) grid coordinates (see below). + +![MODIS Grid](https://modis-land.gsfc.nasa.gov/images/MODIS_sinusoidal_grid1.gif) + +For example, MODIS data right above the equator is all grid coordinates with `v07`. + +```python, catalog_filtering, results='raw' +equator = modis_catalog.where(F.col('gid').like('%v07%')) +equator.select('date', 'gid') \ + .show(10, truncate=False) +``` + +Now that we have prepared our catalog, we simply pass the DataFrame or CSV string to the `raster` DataSource to load the imagery. The `catalog_col_names` parameter gives the columns that contain the URI's to be read. + +```python, read_catalog +rf = spark.read.raster( + catalog=equator, + catalog_col_names=['red', 'nir'], +) +rf.printSchema() +``` + +Observe the schema of the resulting DataFrame has a projected raster struct for each column passed in `catalog_col_names`. For reference, the URI is now in a column appended with `_path`. Taking a quick look at the representation of the data, we see again each row contains an arbitrary portion of the entire scene coverage. We also see that for two-D catalogs, each row contains the same spatial extent for all tiles in that row. + +```python, cat_read_sample, results='raw' +rf.select('gid', rf_extent('red'), rf_extent('nir'), rf_tile('red'), rf_tile('nir')) \ + .show(3, truncate=True) +``` + +## Lazy Raster Reading + +By default, reading raster pixel values is delayed until it is absolutely needed. The DataFrame will contain metadata and pointers to the appropriate portion of the data until reading of the source raster data is absolutely necessary. This can save a significant of computation and I/O time for two reasons. First, a catalog may contain a large number of rows. Second, the `raster` DataSource attempts to apply spatial preciates (e.g. `where`/`WHERE` clauses with `st_intersects`, et al.) at row creation, reducing the chance of unneeded data being fetched. + +Consider the following two reads of the same data source. In the first, the lazy case, there is a pointer to the URI, extent and band to read. This will not be evaluated until the cell values are absolutely required. The second case shows the option to force the raster to be fully loaded right away. + +```python, lazy_demo_1, results='raw' +uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif' +spark.read.raster(uri) \ + .select('proj_raster.tile') \ + .show(1, truncate=False) + +``` + +```python, lazy_demo_2, results='raw' +spark.read.raster(uri, lazy_tiles=False) \ + .select('proj_raster.tile') \ + .show(1, truncate=False) +``` + +In the initial examples on this page, you may have noticed that the realized (non-lazy) _tiles_ are shown, but we did not change `lazy_tiles`. Instead, we used @ref:[`rf_tile`](reference.md#rf-tile) to explicitly request the realized _tile_ from the lazy representation. + +## Multiband Rasters + +A multiband raster represents a three dimensional numeric array. The first two dimensions are spatial, and the third dimension is typically designated for different spectral @ref:[bands](concepts.md#band). The bands could represent intensity of different wavelengths of light (or other electromagnetic radiation), or they could measure other phenomena such as time, quality indications, or additional gas concentrations, etc. + +Multiband rasters files have a strictly ordered set of bands, which are typically indexed from 1. Some files have metadata tags associated with each band. Some files have a color interpetation metadata tag indicating how to interpret the bands. + +When reading a multiband raster or a _catalog_ describing multiband rasters, you will need to know ahead of time which bands you want to read. You will specify the bands to read, **indexed from zero**, as a list of integers into the `band_indexes` parameter of the `raster` reader. + +For example, we can read a four-band (red, green, blue, and near-infrared) image as follows. The individual rows of the resulting DataFrame still represent distinct spatial extents, with a projected raster column for each band specified by `band_indexes`. + +```python, multiband +mb = spark.read.raster('s3://s22s-test-geotiffs/naip/m_3807863_nw_17_1_20160620.tif', + band_indexes=[0, 1, 2, 3], + ) +mb.printSchema() +``` + +If a band is passed into `band_indexes` that exceeds the number of bands in the raster, a projected raster column will still be generated in the schema but the column will be full of `null` values. + +You can also pass a `catalog` and `band_indexes` together into the `raster` reader. This will create a projected raster column for the combination of all items passed into `catalog_col_names` and `band_indexes`. Again if a band in `band_indexes` exceeds the number of bands in a raster, it will have a `null` value for the corresponding column. + +Here is a trivial example with a _catalog_ over multiband rasters. We specify two columns containing URIs and two bands, resulting in four projected raster columns. + +```python, multiband_catalog +import pandas as pd +mb_cat = pd.DataFrame([ + {'foo': 's3://s22s-test-geotiffs/naip/m_3807863_nw_17_1_20160620.tif', + 'bar': 's3://s22s-test-geotiffs/naip/m_3807863_nw_17_1_20160620.tif', + }, +]) +mb2 = spark.read.raster(catalog=spark.createDataFrame(mb_cat), + catalog_col_names=['foo', 'bar'], + band_indexes=[0, 1], + tile_dimensions=(64,64) + ) +mb2.printSchema() +``` + +## GeoTrellis + +### GeoTrellis Catalogs + +[GeoTrellis][GeoTrellis] is one of the key libraries upon which RasterFrames is built. It provides a Scala language API for working with geospatial raster data. GeoTrellis defines a [tile layer storage](https://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html) format for persisting imagery mosaics. RasterFrames provides a DataSource supporting both @ref:[reading](raster-read.md#geotrellis-layers) and @ref:[writing](raster-write.md#geotrellis-layers) GeoTrellis layers. + +A GeoTrellis catalog is a set of GeoTrellis layers. We can read a DataFrame giving details of the content of a catalog using the syntax below. The scheme is typically `hdfs` or `file`, or a cloud storage provider like `s3`. + +```python, geotrellis_cat, evaluate=False +gt_cat = spark.read.geotrellis_catalog('scheme://path-to-gt-catalog') +``` + +### GeoTrellis Layers + +The catalog will give details on the individual layers available for query. We can read each layer with the URI to the catalog, the layer name, and the desired zoom level. + +```python, geotrellis, evaluate=False +gt_layer = spark.read.geotrellis(path='scheme://path-to-gt-catalog', layer=layer_name, zoom=zoom_level) +``` + +This will return a RasterFrame with additional [GeoTrellis-specific metadata](https://geotrellis.readthedocs.io/en/latest/guide/core-concepts.html#layouts-and-tile-layers), inherited from GeoTrellis, stored as JSON in the metadata of the _tile_ column. + +[CRS]: concepts.md#coordinate-reference-system--crs +[Extent]: concepts.md#extent +[Tile]: concepts.md#tile +[GeoTrellis]: https://geotrellis.readthedocs.io/en/latest/ diff --git a/pyrasterframes/src/main/python/docs/raster-write.pymd b/pyrasterframes/src/main/python/docs/raster-write.pymd new file mode 100644 index 000000000..df68c5e8e --- /dev/null +++ b/pyrasterframes/src/main/python/docs/raster-write.pymd @@ -0,0 +1,108 @@ +# Writing Raster Data + +RasterFrames is oriented toward large scale analyses of spatial data. The primary output of these analyses could be a @ref:[statistical summary](aggregation.md), a @ref:[machine learning model](machine-learning.md), or some other result that is generally much smaller than the input dataset. + +However, there are times in any analysis where writing a representative sample of the work in progress provides valuable feedback on the current state of the process and results. + +```python imports, echo=False +import pyrasterframes +from pyrasterframes.rasterfunctions import * +from IPython.display import display +import os.path + +spark = pyrasterframes.get_spark_session() +``` + +## Tile Samples + +We have some convenience methods to quickly visualize _tile_s (see discussion of the RasterFrame @ref:[schema](raster-read.md#single-raster) for orientation to the concept) when inspecting a subset of the data in a Notebook. + +In an IPython or Jupyter interpreter, a `Tile` object will be displayed as an image with limited metadata. + +```python tile_sample +def scene(band): + b = str(band).zfill(2) # converts int 2 to '02' + return 'https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/' \ + 'MCD43A4.A2019059.h11v08.006.2019072203257_B{}.TIF'.format(b) +spark_df = spark.read.raster(scene(2), tile_dimensions=(128, 128)) +tile = spark_df.select(rf_tile('proj_raster').alias('tile')).first()['tile'] +tile +``` + +```python display_tile, echo=False, output=True +display(tile) # IPython.display function +``` + +## DataFrame Samples + +Within an IPython or Jupyter interpreter, a Pandas DataFrame containing a column of _tiles_ will be rendered as the samples discussed above. Simply import the `rf_ipython` submodule to enable enhanced HTML rendering of a Pandas DataFrame. + +In the example below, notice the result is limited to a small subset. For more information about why this is important, see the @ref:[Pandas and NumPy discussion](numpy-pandas.md). + +```python to_pandas, evaluate=True +import pyrasterframes.rf_ipython + +pandas_df = spark_df.select( + rf_extent('proj_raster').alias('extent'), + rf_tile('proj_raster').alias('tile'), + ).limit(5).toPandas() +``` + +Viewing the DataFrame in Jupyter looks like this. + +```python show_pandas, evaluate=False +pandas_df +``` + +@@include[df-samples-output.md](static/df-samples-output.md) + +## GeoTIFFs + +GeoTIFF is one of the most common file formats for spatial data, providing flexibility in data encoding, representation, and storage. RasterFrames provides a specialized Spark DataFrame writer for rendering a RasterFrame to a GeoTIFF. + +One downside to GeoTIFF is that it is not a big data native format. To create a GeoTIFF, all the data to be encoded has to be in the memory of one computer (in Spark parlance, this is a "collect"), limiting it's maximum size substantially compared to that of a full cluster environment. When rendering GeoTIFFs in RasterFrames, you must either specify the dimensions of the output raster, or deliberately limit the size of the collected data. + +Fortunately, we can use the cluster computing capability to downsample the data into a more manageable size. For sake of example, let's render an overview of a scene's red band as a small raster, reprojecting it to latitude and longitude coordinates on the [WGS84](https://en.wikipedia.org/wiki/World_Geodetic_System) reference ellipsoid (aka [EPSG:4326](https://spatialreference.org/ref/epsg/4326/)). + +```python write_geotiff +outfile = os.path.join('/tmp', 'geotiff-overview.tif') +spark_df.write.geotiff(outfile, crs='EPSG:4326', raster_dimensions=(256, 256)) +``` + +We can view the written file with `rasterio`: + +```python view_geotiff +import rasterio +from rasterio.plot import show, show_hist + +with rasterio.open(outfile) as src: + # View raster + show(src, adjust='linear') + # View data distribution + show_hist(src, bins=50, lw=0.0, stacked=False, alpha=0.6, + histtype='stepfilled', title="Overview Histogram") +``` + +If there are many _tile_ or projected raster columns in the DataFrame, the GeoTIFF writer will write each one as a separate band in the file. Each band in the output will be tagged the input column names for reference. + +```python, echo=False +os.remove(outfile) +``` + +## GeoTrellis Layers + +[GeoTrellis][GeoTrellis] is one of the key libraries upon which RasterFrames is built. It provides a Scala language API for working with geospatial raster data. GeoTrellis defines a [tile layer storage](https://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html) format for persisting imagery mosaics. RasterFrames can write data from a `RasterFrameLayer` into a [GeoTrellis Layer](https://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html). RasterFrames provides a `geotrellis` DataSource that supports both @ref:[reading](raster-read.md#geotrellis-layers) and @ref:[writing](raster-write.md#geotrellis-layers) GeoTrellis layers. + +> An example is forthcoming. In the mean time referencing the [`GeoTrellisDataSourceSpec` test code](https://github.com/locationtech/rasterframes/blob/develop/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala) may help. + +## Parquet + +You can write a RasterFrame to the [Apache Parquet][Parquet] format. This format is designed to efficiently persist and query columnar data in distributed file system, such as HDFS. It also provides benefits when working in single node (or "local") mode, such as tailoring organization for defined query patterns. + +```python write_parquet, evaluate=False +spark_df.withColumn('exp', rf_expm1('proj_raster')) \ + .write.mode('append').parquet('hdfs:///rf-user/sample.pq') +``` + +[GeoTrellis]: https://geotrellis.readthedocs.io/en/latest/ +[Parquet]: https://spark.apache.org/docs/latest/sql-data-sources-parquet.html diff --git a/pyrasterframes/src/main/python/docs/reference.pymd b/pyrasterframes/src/main/python/docs/reference.pymd new file mode 100644 index 000000000..bc4ae3413 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/reference.pymd @@ -0,0 +1,629 @@ +# Function Reference + +RasterFrames provides a rich set of columnar function for processing geospatial raster data. In Spark SQL, the functions are already registered in the SQL engine; they are usually prefixed with `rf_`. In Python, they are available in the `pyrasterframes.rasterfunctions` module. + +The convention in this document will be to define the function signature as below, with its return type, the function name, and named arguments with their types. + +``` +ReturnDataType function_name(InputDataType argument1, InputDataType argument2) +``` + +For the Scala documentation on these functions, see @scaladoc[`RasterFunctions`][RasterFunctions]. The full Scala API documentation can be found [here][scaladoc]. + +## List of Available SQL and Python Functions + +@@toc { depth=3 } + + +To import RasterFrames functions into the environment, import from `pyrasterframes.rasterfunctions`. + +```python +from pyrasterframes.rasterfunctions import * +``` + +Functions starting with `rf_`, which are for raster, and `st_`, which are for vector geometry, +become available for use with DataFrames. You can view all of the available functions with the following. + +```python, evaluate=False +[fn for fn in dir() if fn.startswith('rf_') or fn.startswith('st_')] +``` + +## Vector Operations + +Various LocationTech GeoMesa user-defined functions (UDFs) dealing with `geomtery` type columns are provided in the SQL engine and within the `pyrasterframes.rasterfunctions` Python module. These are documented in the [LocationTech GeoMesa Spark SQL documentation](https://www.geomesa.org/documentation/user/spark/sparksql_functions.html#). These functions are all prefixed with `st_`. + +RasterFrames provides some additional functions for vector geometry operations. + +### st_reproject + + Geometry st_reproject(Geometry geom, String origin_crs, String destination_crs) + + +Reproject the vector `geom` from `origin_crs` to `destination_crs`. Both `_crs` arguments are either [proj4](https://proj4.org/usage/quickstart.html) strings, [EPSG codes](https://www.epsg-registry.org/) or [OGC WKT](https://www.opengeospatial.org/standards/wkt-crs) for coordinate reference systems. + + +### st_extent + + Struct[Double xmin, Double xmax, Double ymin, Double ymax] st_extent(Geometry geom) + +Extracts the bounding box (extent/envelope) of the geometry. + +See also GeoMesa [st_envelope](https://www.geomesa.org/documentation/user/spark/sparksql_functions.html#st-envelope) which returns a Geometry type. + +### st_geometry + + Geometry st_geometry(Struct[Double xmin, Double xmax, Double ymin, Double ymax] extent) + +Convert an extent to a Geometry. The extent likely comes from @ref:[`st_extent`](reference.md#st-extent) or @ref:[`rf_extent`](reference.md#rf-extent). + +## Tile Metadata and Mutation + +Functions to access and change the particulars of a `tile`: its shape and the data type of its cells. See section on @ref:["NoData" handling](nodata-handling.md) for additional discussion of cell types. + +### rf_dimensions + + Struct[Int, Int] rf_dimensions(Tile tile) + +Get number of columns and rows in the `tile`, as a Struct of `cols` and `rows`. + +### rf_cell_type + + Struct[String] rf_cell_type(Tile tile) + +Get the cell type of the `tile`. The cell type can be changed with @ref:[rf_convert_cell_type](reference.md#rf-convert-cell-type). + +### rf_tile + + Tile rf_tile(ProjectedRasterTile proj_raster) + +Get the fully realized (non-lazy) `tile` from a `ProjectedRasterTile` struct column. + +### rf_extent + + Struct[Double xmin, Double xmax, Double ymin, Double ymax] rf_extent(ProjectedRasterTile proj_raster) + Struct[Double xmin, Double xmax, Double ymin, Double ymax] rf_extent(RasterSource proj_raster) + +Fetches the extent (bounding box or envelope) of a `ProjectedRasterTile` or `RasterSource` type tile columns. + +### rf_crs + + Struct rf_crs(ProjectedRasterTile proj_raster) + Struct rf_crs(RasterSource proj_raster) + +Fetch CRS structure representing the coordinate reference system of a `ProjectedRasterTile` or `RasterSource` type tile columns. + +### rf_mk_crs + + Struct rf_mk_crs(String crsText) + +Construct a CRS structure from one of its string representations. Three forms are supported: + +* [EPSG code](https://www.epsg-registry.org/): `EPSG:` +* [Proj4 string](https://proj.org/): `+proj ` +* [WKT String](http://www.geoapi.org/3.0/javadoc/org/opengis/referencing/doc-files/WKT.html) with embedded EPSG code: `GEOGCS["", , , {,} {,}]` + +Example: `SELECT rf_mk_crs('EPSG:4326')` + +### rf_convert_cell_type + + Tile rf_convert_cell_type(Tile tileCol, String cellType) + +Convert `tileCol` to a different cell type. Available cell types can be retrieved with the @ref:[rf_cell_types](reference.md#rf-cell-types) function. + +### rf_resample + + Tile rf_resample(Tile tile, Double factor) + Tile rf_resample(Tile tile, Int factor) + Tile rf_resample(Tile tile, Tile shape_tile) + + +Change the tile dimension. Passing a numeric `factor` will scale the number of columns and rows in the tile: 1.0 is the same number of columns and row; less than one downsamples the tile; and greater than one upsamples the tile. Passing a `shape_tile` as the second argument outputs `tile` having the same number of columns and rows as `shape_tile`. All resampling is by nearest neighbor method. + +## Tile Creation + +Functions to create a new Tile column, either from scratch or from existing data not yet in a `tile`. + +### rf_make_zeros_tile + +``` +Tile rf_make_zeros_tile(Int tile_columns, Int tile_rows, [CellType cell_type]) +Tile rf_make_zeros_tile(Int tile_columns, Int tile_rows, [String cell_type_name]) +``` + +Create a `tile` of shape `tile_columns` by `tile_rows` full of zeros, with the optional cell type; default is float64. See @ref:[this discussion](nodata-handling.md#cell-types) on cell types for info on the `cell_type` argument. All arguments are literal values and not column expressions. + +### rf_make_ones_tile + +``` +Tile rf_make_ones_tile(Int tile_columns, Int tile_rows, [CellType cell_type]) +Tile rf_make_ones_tile(Int tile_columns, Int tile_rows, [String cell_type_name]) +``` + + +Create a `tile` of shape `tile_columns` by `tile_rows` full of ones, with the optional cell type; default is float64. See @ref:[this discussion](nodata-handling.md#cell-types) on cell types for info on the `cell_type` argument. All arguments are literal values and not column expressions. + +### rf_make_constant_tile + + Tile rf_make_constant_tile(Numeric constant, Int tile_columns, Int tile_rows, [CellType cell_type]) + Tile rf_make_constant_tile(Numeric constant, Int tile_columns, Int tile_rows, [String cell_type_name]) + + +Create a `tile` of shape `tile_columns` by `tile_rows` full of `constant`, with the optional cell type; default is float64. See @ref:[this discussion](nodata-handling.md#cell-types) on cell types for info on the `cell_type` argument. All arguments are literal values and not column expressions. + + +### rf_rasterize + + Tile rf_rasterize(Geometry geom, Geometry tile_bounds, Int value, Int tile_columns, Int tile_rows) + + +Convert a vector Geometry `geom` into a Tile representation. The `value` will be "burned-in" to the returned `tile` where the `geom` intersects the `tile_bounds`. Returned `tile` will have shape `tile_columns` by `tile_rows`. Values outside the `geom` will be assigned a NoData value. Returned `tile` has cell type `int32`, note that `value` is of type Int. + +Parameters `tile_columns` and `tile_rows` are literals, not column expressions. The others are column expressions. + + +Example use. In the code snip below, you can visualize the `tri` and `b` geometries with tools like [Wicket](https://arthur-e.github.io/Wicket/sandbox-gmaps3.html). The result is a right triangle burned into the `tile`, with NoData values shown as ∘. + + +```python +spark.sql(""" +SELECT rf_render_ascii( + rf_rasterize(tri, b, 8, 10, 10)) + +FROM + ( SELECT st_geomFromWKT('POLYGON((1.5 0.5, 1.5 1.5, 0.5 0.5, 1.5 0.5))') AS tri, + st_geomFromWKT('POLYGON((0.0 0.0, 2.0 0.0, 2.0 2.0, 0.0 2.0, 0.0 0.0))') AS b + ) r +""").show(1, False) +``` + + +### rf_array_to_tile + + Tile rf_array_to_tile(Array arrayCol, Int numCols, Int numRows) + +Python only. Create a `tile` from a Spark SQL [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), filling values in row-major order. + +### rf_assemble_tile + + Tile rf_assemble_tile(Int colIndex, Int rowIndex, Numeric cellData, Int numCols, Int numRows, [CellType cell_type]) + Tile rf_assemble_tile(Int colIndex, Int rowIndex, Numeric cellData, Int numCols, Int numRows, [String cell_type_name]) + +SQL: `Tile rf_assemble_tile(Int colIndex, Int rowIndex, Numeric cellData, Int numCols, Int numRows)` + +Create `tile`s of dimension `numCols` by `numRows` from a column of cell data with location indices. This function is the inverse of @ref:[`rf_explode_tiles`](reference.md#rf-explode-tiles). Intended use is with a `groupby`, producing one row with a new `tile` per group. In Python, the `numCols`, `numRows` and `cellType` arguments are literal values, others are column expressions. +See @ref:[this discussion](nodata-handling.md#cell-types) on cell types for info on the optional `cell_type` argument. The default is float64. +SQL implementation does not accept a cell_type argument. It returns a float64 cell type `tile` by default. + +## Masking and NoData + +See @ref:[NoData handling](nodata-handling.md) for conceptual discussion of cell types and NoData. + +There are statistical functions of the count of data and NoData values per `tile` and aggregate over a `tile` column: @ref:[`rf_data_cells`](reference.md#rf-data-cells), @ref:[`rf_no_data_cells`](reference.md#rf-no-data-cells), @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells), and @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells). + +Masking is a raster operation that sets specific cells to NoData based on the values in another raster. + +### rf_mask + + Tile rf_mask(Tile tile, Tile mask) + +Where the `mask` contains NoData, replace values in the `tile` with NoData. + +Returned `tile` cell type will be coerced to one supporting NoData if it does not already. + +See also @[`rf_rasterize`](reference.md#rf-rasterize). + + +### rf_inverse_mask + + Tile rf_inverse_mask(Tile tile, Tile mask) + +Where the `mask` _does not_ contain NoData, replace values in `tile` with NoData. + +### rf_mask_by_value + + Tile rf_mask_by_value(Tile data_tile, Tile mask_tile, Int mask_value) + +Generate a `tile` with the values from `data_tile`, with NoData in cells where the `mask_tile` is equal to `mask_value`. + + +### rf_is_no_data_tile + + Boolean rf_is_no_data_tile(Tile) + +Returns true if `tile` contains only NoData. By definition returns false if cell type does not support NoData. To count NoData cells or data cells, see @ref:[`rf_no_data_cells`](reference.md#rf-no-data-cells), @ref:[`rf_data_cells`](reference.md#rf-data-cells), @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells), @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells), @ref:[`rf_agg_local_no_data_cells`](reference.md#rf-agg-local-no-data-cells), and @ref:[`rf_agg_local_data_cells`](reference.md#rf-agg-local-data-cells). This function is distinguished from @ref:[`rf_for_all`](reference.md#rf-for-all), which tests that values are not NoData and nonzero. + +### rf_with_no_data + + Tile rf_with_no_data(Tile tile, Double no_data_value) + +Python only. Return a `tile` column marking as NoData all cells equal to `no_data_value`. + +The `no_data_value` argument is a literal Double, not a Column expression. + +If input `tile` had a NoData value already, the behaviour depends on if its cell type is floating point or not. For floating point cell type `tile`, NoData values on the input `tile` remain NoData values on the output. For integral cell type `tile`s, the previous NoData values become literal values. + +## Local Map Algebra + +[Local map algebra](https://gisgeography.com/map-algebra-global-zonal-focal-local/) raster operations are element-wise operations on a single tile (unary), between a `tile` and a scalar, between two `tile`s, or across many `tile`s. + +When these operations encounter a NoData value in either operand, the cell in the resulting `tile` will have a NoData. + +The binary local map algebra functions have similar variations in the Python API depending on the left hand side type: + + - `rf_local_op`: applies `op` to two columns; the right hand side can be a `tile` or a numeric column. + - `rf_local_op_double`: applies `op` to a `tile` and a literal scalar, coercing the `tile` to a floating point type + - `rf_local_op_int`: applies `op` to a `tile` and a literal scalar, without coercing the `tile` to a floating point type + +The SQL API does not require the `rf_local_op_double` or `rf_local_op_int` forms (just `rf_local_op`). + +Local map algebra operations for more than two `tile`s are implemented to work across rows in the DataFrame. As such, they are @ref:[aggregate functions](reference.md#tile-local-aggregate-statistics). + +### rf_local_add + + Tile rf_local_add(Tile tile1, Tile rhs) + Tile rf_local_add_int(Tile tile1, Int rhs) + Tile rf_local_add_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise sum of `tile1` and `rhs`. + +### rf_local_subtract + + Tile rf_local_subtract(Tile tile1, Tile rhs) + Tile rf_local_subtract_int(Tile tile1, Int rhs) + Tile rf_local_subtract_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise difference of `tile1` and `rhs`. + + +### rf_local_multiply + + Tile rf_local_multiply(Tile tile1, Tile rhs) + Tile rf_local_multiply_int(Tile tile1, Int rhs) + Tile rf_local_multiply_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise product of `tile1` and `rhs`. This is **not** the matrix multiplication of `tile1` and `rhs`. + + +### rf_local_divide + + Tile rf_local_divide(Tile tile1, Tile rhs) + Tile rf_local_divide_int(Tile tile1, Int rhs) + Tile rf_local_divide_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise quotient of `tile1` and `rhs`. + + +### rf_normalized_difference + + Tile rf_normalized_difference(Tile tile1, Tile tile2) + + +Compute the normalized difference of the the two `tile`s: `(tile1 - tile2) / (tile1 + tile2)`. Result is always floating point cell type. This function has no scalar variant. + +### rf_local_less + + Tile rf_local_less(Tile tile1, Tile rhs) + Tile rf_local_less_int(Tile tile1, Int rhs) + Tile rf_local_less_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise evaluation of `tile1` is less than `rhs`. + +### rf_local_less_equal + + Tile rf_local_less_equal(Tile tile1, Tile rhs) + Tile rf_local_less_equal_int(Tile tile1, Int rhs) + Tile rf_local_less_equal_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise evaluation of `tile1` is less than or equal to `rhs`. + +### rf_local_greater + + Tile rf_local_greater(Tile tile1, Tile rhs) + Tile rf_local_greater_int(Tile tile1, Int rhs) + Tile rf_local_greater_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise evaluation of `tile1` is greater than `rhs`. + +### rf_local_greater_equal + + Tile rf_local_greater_equal(Tile tile1, Tile rhs) + Tile rf_local_greater_equal_int(Tile tile1, Int rhs) + Tile rf_local_greater_equal_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise evaluation of `tile1` is greater than or equal to `rhs`. + +### rf_local_equal + + Tile rf_local_equal(Tile tile1, Tile rhs) + Tile rf_local_equal_int(Tile tile1, Int rhs) + Tile rf_local_equal_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise equality of `tile1` and `rhs`. + +### rf_local_unequal + + Tile rf_local_unequal(Tile tile1, Tile rhs) + Tile rf_local_unequal_int(Tile tile1, Int rhs) + Tile rf_local_unequal_double(Tile tile1, Double rhs) + + +Returns a `tile` column containing the element-wise inequality of `tile1` and `rhs`. + +### rf_round + + Tile rf_round(Tile tile) + +Round cell values to the nearest integer without changing the cell type. + +### rf_abs + + Tile rf_abs(Tile tile) + + Compute the absolute value of cell value. + +### rf_exp + + Tile rf_exp(Tile tile) + +Performs cell-wise exponential. + +### rf_exp10 + + Tile rf_exp10(Tile tile) + +Compute 10 to the power of cell values. + +### rf_exp2 + + Tile rf_exp2(Tile tile) + +Compute 2 to the power of cell values. + +### rf_expm1 + + Tile rf_expm1(Tile tile) + +Performs cell-wise exponential, then subtract one. Inverse of @ref:[`log1p`](reference.md#log1p). + +### rf_log + + Tile rf_log(Tile tile) + +Performs cell-wise natural logarithm. + +### rf_log10 + + Tile rf_log10(Tile tile) + +Performs cell-wise logarithm with base 10. + +### rf_log2 + + Tile rf_log2(Tile tile) + +Performs cell-wise logarithm with base 2. + +### rf_log1p + + Tile rf_log1p(Tile tile) + +Performs natural logarithm of cell values plus one. Inverse of @ref:[`rf_expm1`](reference.md#rf-expm1). + +## Tile Statistics + +The following functions compute a statistical summary per row of a `tile` column. The statistics are computed across the cells of a single `tile`, within each DataFrame Row. + +### rf_tile_sum + + Double rf_tile_sum(Tile tile) + + +Computes the sum of cells in each row of column `tile`, ignoring NoData values. + +### rf_tile_mean + + Double rf_tile_mean(Tile tile) + + +Computes the mean of cells in each row of column `tile`, ignoring NoData values. + + +### rf_tile_min + + Double rf_tile_min(Tile tile) + + +Computes the min of cells in each row of column `tile`, ignoring NoData values. + + +### rf_tile_max + + Double rf_tile_max(Tile tile) + + +Computes the max of cells in each row of column `tile`, ignoring NoData values. + + +### rf_no_data_cells + + Long rf_no_data_cells(Tile tile) + + +Return the count of NoData cells in the `tile`. + +### rf_data_cells + + Long rf_data_cells(Tile tile) + +Return the count of data cells in the `tile`. + +### rf_exists + + Boolean rf_exists(Tile tile) + +Returns true if any cells in the tile are true (non-zero and not NoData). + +### rf_for_all + + Boolean rf_for_all(Tile tile) + +Returns true if all cells in the tile are true (non-zero and not NoData). See also @ref:[`rf_is_no_data_tile](reference.md#rf-is-no-data-tile), which tests that all cells are NoData. + +### rf_tile_stats + + Struct[Long, Long, Double, Double, Double, Double] rf_tile_stats(Tile tile) + +Computes the following statistics of cells in each row of column `tile`: data cell count, NoData cell count, minimum, maximum, mean, and variance. The minimum, maximum, mean, and variance are computed ignoring NoData values. Resulting column has the below schema. + +```python echo=False +spark.sql("SELECT rf_tile_stats(rf_make_ones_tile(5, 5, 'float32')) as tile_stats").printSchema() +``` + +### rf_tile_histogram + + Struct[Array[Struct[Double, Long]]] rf_tile_histogram(Tile tile) + +Computes a count of cell values within each row of `tile`. The `bins` array is of tuples of histogram values and counts. Typically values are plotted on the x-axis and counts on the y-axis. Resulting column has the below schema. Related is the @ref:[`rf_agg_approx_histogram`](reference.md#rf-agg-approx-histogram) which computes the statistics across all rows in a group. + +```python echo=False +spark.sql("SELECT rf_tile_histogram(rf_make_ones_tile(5, 5, 'float32')) as tile_histogram").printSchema() +``` + +## Aggregate Tile Statistics + +These functions compute statistical summaries over all of the cell values *and* across all the rows in the DataFrame or group. + +### rf_agg_mean + + Double rf_agg_mean(Tile tile) + +_SQL_: @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`(tile).mean` + +Aggregates over the `tile` and return the mean of cell values, ignoring NoData. Equivalent to @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`.mean`. + + +### rf_agg_data_cells + + Long rf_agg_data_cells(Tile tile) + +_SQL_: @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`(tile).dataCells` + +Aggregates over the `tile` and return the count of data cells. Equivalent to @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`.dataCells`. + +### rf_agg_no_data_cells + + Long rf_agg_no_data_cells(Tile tile) + +_SQL_: @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`(tile).dataCells` + +Aggregates over the `tile` and return the count of NoData cells. Equivalent to @ref:[`rf_agg_stats`](reference.md#rf-agg-stats)`.noDataCells`. C.F. @ref:[`rf_no_data_cells`](reference.md#rf-no-data-cells) a row-wise count of no data cells. + +### rf_agg_stats + + Struct[Long, Long, Double, Double, Double, Double] rf_agg_stats(Tile tile) + + +Aggregates over the `tile` and returns statistical summaries of cell values: number of data cells, number of NoData cells, minimum, maximum, mean, and variance. The minimum, maximum, mean, and variance ignore the presence of NoData. + +### rf_agg_approx_histogram + + Struct[Array[Struct[Double, Long]]] rf_agg_approx_histogram(Tile tile) + + +Aggregates over all of the rows in DataFrame of `tile` and returns a count of each cell value to create a histogram with values are plotted on the x-axis and counts on the y-axis. Related is the @ref:[`rf_tile_histogram`](reference.md#rf-tile-histogram) function which operates on a single row at a time. + + +## Tile Local Aggregate Statistics + +Local statistics compute the element-wise statistics across a DataFrame or group of `tile`s, resulting in a `tile` that has the same dimension. + +When these functions encounter NoData in a cell location, it will be ignored. + +### rf_agg_local_max + + Tile rf_agg_local_max(Tile tile) + +Compute the cell-local maximum operation over `tile`s in a column. + +### rf_agg_local_min + + Tile rf_agg_local_min(Tile tile) + +Compute the cell-local minimum operation over `tile`s in a column. + +### rf_agg_local_mean + + Tile rf_agg_local_mean(Tile tile) + +Compute the cell-local mean operation over `tile`s in a column. + +### rf_agg_local_data_cells + + Tile rf_agg_local_data_cells(Tile tile) + +Compute the cell-local count of data cells over `tile`s in a column. Returned `tile` has a cell type of `int32`. + +### rf_agg_local_no_data_cells + + Tile rf_agg_local_no_data_cells(Tile tile) + +Compute the cell-local count of NoData cells over `tile`s in a column. Returned `tile` has a cell type of `int32`. + +### rf_agg_local_stats + + Struct[Tile, Tile, Tile, Tile, Tile] rf_agg_local_stats(Tile tile) + +Compute cell-local aggregate count, minimum, maximum, mean, and variance for a column of `tile`s. Returns a struct of five `tile`s. + + +## Converting Tiles + +RasterFrames provides several ways to convert a `tile` into other data structures. See also functions for @ref:[creating tiles](reference.md#tile-creation). + +### rf_explode_tiles + + Int, Int, Numeric* rf_explode_tiles(Tile* tile) + +Create a row for each cell in `tile` columns. Many `tile` columns can be passed in, and the returned DataFrame will have one numeric column per input. There will also be columns for `column_index` and `row_index`. Inverse of @ref:[`rf_assemble_tile`](reference.md#rf-assemble-tile). When using this function, be sure to have a unique identifier for rows in order to successfully invert the operation. + +### rf_explode_tiles_sample + + Int, Int, Numeric* rf_explode_tiles_sample(Double sample_frac, Long seed, Tile* tile) + +Python only. As with @ref:[`rf_explode_tiles`](reference.md#rf-explode-tiles), but taking a randomly sampled subset of cells. Equivalent to the `rf_explode-tiles`, but allows a random subset of the data to be selected. Parameter `sample_frac` should be between 0.0 and 1.0. + +### rf_tile_to_array_int + + Array rf_tile_to_array_int(Tile tile) + +Convert Tile column to Spark SQL [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), in row-major order. Float cell types will be coerced to integral type by flooring. + +### rf_tile_to_array_double + + Array rf_tile_to_arry_double(Tile tile) + +Convert tile column to Spark [Array](http://spark.apache.org/docs/2.3.2/api/python/pyspark.sql.html#pyspark.sql.types.ArrayType), in row-major order. Integral cell types will be coerced to floats. + +### rf_render_ascii + + String rf_render_ascii(Tile tile) + +Pretty print the tile values as plain text. + +### rf_render_matrix + + String rf_render_matrix(Tile tile) + +Render Tile cell values as numeric values, for debugging purposes. + +[RasterFunctions]: org.locationtech.rasterframes.RasterFunctions +[scaladoc]: latest/api/index.html diff --git a/pyrasterframes/src/main/python/docs/static/df-samples-output.md b/pyrasterframes/src/main/python/docs/static/df-samples-output.md new file mode 100644 index 000000000..7cbfaf1ae --- /dev/null +++ b/pyrasterframes/src/main/python/docs/static/df-samples-output.md @@ -0,0 +1,51 @@ +
                          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                          extenttile
                          0(-7783653.637667, 1052646.4919514267, -7724349.609951426, 1111950.519667)Image
                          1(-7724349.609951427, 1052646.4919514267, -7665045.582235852, 1111950.519667)Image
                          2(-7665045.582235853, 1052646.4919514267, -7605741.554520279, 1111950.519667)Image
                          3(-7605741.55452028, 1052646.4919514267, -7546437.526804706, 1111950.519667)Image
                          4(-7546437.526804707, 1052646.4919514267, -7487133.499089133, 1111950.519667)Image
                          +
                          diff --git a/pyrasterframes/src/main/python/docs/static/rasterframe-anatomy.png b/pyrasterframes/src/main/python/docs/static/rasterframe-anatomy.png new file mode 100644 index 000000000..08abab9f2 Binary files /dev/null and b/pyrasterframes/src/main/python/docs/static/rasterframe-anatomy.png differ diff --git a/pyrasterframes/src/main/python/docs/static/rasterframe-sample.md b/pyrasterframes/src/main/python/docs/static/rasterframe-sample.md new file mode 100644 index 000000000..c7ec4ed5d --- /dev/null +++ b/pyrasterframes/src/main/python/docs/static/rasterframe-sample.md @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + \ + + + + + + + + + + + + + + + + + + + + + \ + + + + + + + \ + + +
                          timestampcrsextenttile
                          02019-02-28(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(-7783653.637667, 993342.4642358534, -7665045.582235852, 1111950.519667)
                          12019-02-28(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(-7665045.582235853, 993342.4642358534, -7546437.526804706, 1111950.519667)
                          22019-02-28(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(-7546437.526804707, 993342.4642358534, -7427829.471373559, 1111950.519667)
                          32019-02-28(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(-7427829.47137356, 993342.4642358534, -7309221.415942413, 1111950.519667)
                          42019-02-28(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(-7309221.415942414, 993342.4642358534, -7190613.360511266, 1111950.519667)
                          diff --git a/pyrasterframes/src/main/python/docs/static/rasterframes-data-sources.png b/pyrasterframes/src/main/python/docs/static/rasterframes-data-sources.png new file mode 100644 index 000000000..ad2c8b0da Binary files /dev/null and b/pyrasterframes/src/main/python/docs/static/rasterframes-data-sources.png differ diff --git a/pyrasterframes/src/main/python/docs/static/rasterframes-locationtech-stack.png b/pyrasterframes/src/main/python/docs/static/rasterframes-locationtech-stack.png new file mode 100644 index 000000000..869d61742 Binary files /dev/null and b/pyrasterframes/src/main/python/docs/static/rasterframes-locationtech-stack.png differ diff --git a/pyrasterframes/src/main/python/docs/static/sentinel-2-scene-classification-labels.png b/pyrasterframes/src/main/python/docs/static/sentinel-2-scene-classification-labels.png new file mode 100644 index 000000000..f7c29387d Binary files /dev/null and b/pyrasterframes/src/main/python/docs/static/sentinel-2-scene-classification-labels.png differ diff --git a/pyrasterframes/src/main/python/docs/supervised-learning.pymd b/pyrasterframes/src/main/python/docs/supervised-learning.pymd new file mode 100644 index 000000000..e3f73ace4 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/supervised-learning.pymd @@ -0,0 +1,233 @@ +# Supervised Machine Learning + +In this example we will demonstrate how to fit and score a supervised learning model with a sample of Sentinel-2 data and hand-drawn vector labels over different [land cover](https://en.wikipedia.org/wiki/Land_cover) types. + +```python, setup, echo=False +from IPython.core.display import display +from pyrasterframes.utils import create_rf_spark_session +from pyrasterframes.rasterfunctions import * +import pyrasterframes.rf_ipython +from pyspark.sql.functions import lit +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +from docs import resource_dir_uri + +import os + +spark = create_rf_spark_session() +``` + +## Create and Read Raster Catalog + +The first step is to create a Spark DataFrame containing our imagery data. To achieve that we will create @ref:[a catalog DataFrame](raster-catalogs.md#creating-a-catalog). In the catalog, each row represents a distinct area and time, and each column is the URI to a band's image product. In this example our catalog just has one row. After reading the catalog, the resulting Spark DataFrame may have many rows per URI, with a column corresponding to each band. + +The imagery for feature data will come from [eleven bands of 60 meter resolution Sentinel-2](https://earth.esa.int/web/sentinel/user-guides/sentinel-2-msi/resolutions/spatial) imagery. We also will use the [scene classification (SCL)](https://earth.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm) data to identify high quality, non-cloudy pixels. + +```python, read_bands +uri_base = 's3://s22s-test-geotiffs/luray_snp/{}.tif' +bands = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B11', 'B12'] +cols = ['SCL'] + bands + +catalog_df = pd.DataFrame([ + {b: uri_base.format(b) for b in cols} +]) + +df = spark.read.raster(catalog=catalog_df, + catalog_col_names=cols, + tile_dimensions=(128, 128) + ).repartition(100) + +df = df.select( + rf_crs(df.B01).alias('crs'), + rf_extent(df.B01).alias('extent'), + rf_tile(df.SCL).alias('scl'), + rf_tile(df.B01).alias('B01'), + rf_tile(df.B02).alias('B02'), + rf_tile(df.B03).alias('B03'), + rf_tile(df.B04).alias('B04'), + rf_tile(df.B05).alias('B05'), + rf_tile(df.B06).alias('B06'), + rf_tile(df.B07).alias('B07'), + rf_tile(df.B08).alias('B08'), + rf_tile(df.B09).alias('B09'), + rf_tile(df.B11).alias('B11'), + rf_tile(df.B12).alias('B12'), +) +df.printSchema() +``` + +## Data Prep + +### Label Data + +The land classification labels are based on a small set of hand drawn polygons in the GeoJSON file [here](https://github.com/locationtech/rasterframes/blob/develop/pyrasterframes/src/test/resources/luray-labels.geojson). The property `id` indicates the type of land cover in each area. For these integer values, 1 is forest, 2 is cropland, and 3 is developed areas. + +We will create a very small Spark DataFrame of the label shapes and then join it to the raster DataFrame. Such joins are typically expensive, but in this case both datasets are quite small. To speed up the join for the small vector DataFrame, we put the `broadcast` hint on it, which will tell Spark to put a copy of it on each Spark executor. + +After the raster and vector data are joined, we will convert the vector shapes into _tiles_ using the @ref:[`rf_rasterize`](reference.md#rf-rasterize) function. This procedure is sometimes called "burning in" a geometry into a raster. The values in the resulting _tile_ cells are the `id` property of the GeoJSON, which we will use as labels in our supervised learning task. In areas where the geometry does not intersect, the cells will contain NoData. + +```python join_and_rasterize +crses = df.select('crs.crsProj4').distinct().collect() +print('Found ', len(crses), 'distinct CRS.') +crs = crses[0][0] + +label_df = spark.read.geojson(os.path.join(resource_dir_uri(), 'luray-labels.geojson')) \ + .select('id', st_reproject('geometry', lit('EPSG:4326'), lit(crs)).alias('geometry')) \ + .hint('broadcast') + +df_joined = df.join(label_df, st_intersects(st_geometry('extent'), 'geometry')) + +df_joined.createOrReplaceTempView('df_joined') +df_labeled = spark.sql(""" +SELECT *, rf_rasterize(geometry, st_geometry(extent), id, rf_dimensions(B01).cols, rf_dimensions(B01).rows) AS label +FROM df_joined +""") +``` + +## Masking Poor Quality Cells + +To filter only for good quality pixels, we follow roughly the same procedure as demonstrated in the @ref:[quality masking](nodata-handling.md#masking) section of the chapter on NoData. Instead of actually setting NoData values in the unwanted cells of any of the imagery bands, we will just filter out the mask cell values later in the process. + +```python, make_mask +from pyspark.sql.functions import lit + +mask_part = df_labeled.withColumn('nodata', rf_local_equal('scl', lit(0))) \ + .withColumn('defect', rf_local_equal('scl', lit(1))) \ + .withColumn('cloud8', rf_local_equal('scl', lit(8))) \ + .withColumn('cloud9', rf_local_equal('scl', lit(9))) \ + .withColumn('cirrus', rf_local_equal('scl', lit(10))) + +df_mask_inv = mask_part.withColumn('mask', rf_local_add('nodata', 'defect')) \ + .withColumn('mask', rf_local_add('mask', 'cloud8')) \ + .withColumn('mask', rf_local_add('mask', 'cloud9')) \ + .withColumn('mask', rf_local_add('mask', 'cirrus')) \ + .drop('nodata', 'defect', 'cloud8', 'cloud9', 'cirrus') +# at this point the mask contains 0 for good cells and 1 for defect, etc +# convert cell type and set value 1 to NoData +df_mask = df_mask_inv.withColumn('mask', + rf_with_no_data(rf_convert_cell_type('mask', 'uint8'), 1.0) +) + +df_mask.printSchema() +``` + +## Create ML Pipeline + +We import various Spark components that we need to construct our [Pipeline](https://spark.apache.org/docs/latest/ml-pipeline.html). These are the objects that will work in sequence to conduct the data preparation and modeling. + +```python, imports, echo=True +from pyrasterframes import TileExploder +from pyrasterframes.rf_types import NoDataFilter + +from pyspark.ml.feature import VectorAssembler +from pyspark.ml.classification import DecisionTreeClassifier +from pyspark.ml.evaluation import MulticlassClassificationEvaluator +from pyspark.ml import Pipeline +``` + +SparkML requires that each observation be in its own row, and those observations be packed into a single [`Vector`](https://spark.apache.org/docs/latest/api/python/pyspark.ml.html#module-pyspark.ml.linalg) object. The first step is to "explode" the _tiles_ into a single row per cell or pixel with the `TileExploder` (see also @ref:[`rf_explode_tiles`](reference.md#rf_explode_tiles)). If a _tile_ cell contains a NoData it will become a null value after the exploder stage. Then we use the `NoDataFilter` to filter out any rows that missing or null values, which will cause an error during training. Finally we use the SparkML `VectorAssembler` to create that `Vector`. + +Recall above we set undesirable pixels to NoData, so the `NoDataFilter` will remove them at this stage. We apply the filter to the `mask` column and the `label` column, the latter being used during training. When it is time to score the model, the pipeline will ignore the fact that there is no `label` column on the input DataFrame. + +```python, transformers +exploder = TileExploder() + +noDataFilter = NoDataFilter() \ + .setInputCols(['label', 'mask']) + +assembler = VectorAssembler() \ + .setInputCols(bands) \ + .setOutputCol("features") +``` + +We are going to use a decision tree for classification. You can swap out one of the other multi-class classification algorithms if you like. With the algorithm selected we can assemble our modeling pipeline. + +```python, pipeline +classifier = DecisionTreeClassifier() \ + .setLabelCol('label') \ + .setFeaturesCol(assembler.getOutputCol()) + +pipeline = Pipeline() \ + .setStages([exploder, noDataFilter, assembler, classifier]) + +pipeline.getStages() +``` + +## Train the Model + +The next step is to actually run each step of the Pipeline we created, including fitting the decision tree model. We filter the DataFrame for only _tiles_ intersecting the label raster because the label shapes are relatively sparse over the imagery. It would be logically equivalent to either include or exclude thi step, but it is more efficient to filter because it will mean less data going into the pipeline. + +```python, train +model = pipeline.fit(df_mask.filter(rf_tile_sum('label') > 0).cache()) +``` + +## Model Evaluation + +To view the model's performance, we first call the pipeline's `transform` method on the training dataset. This transformed dataset will have the model's prediction included in each row. We next construct an evaluator and pass it the transformed dataset to easily compute the performance metric. We can also create custom metrics using a variety of DataFrame or SQL transformations. + +```python eval +prediction_df = model.transform(df_mask) \ + .drop(assembler.getOutputCol()).cache() +prediction_df.printSchema() + +eval = MulticlassClassificationEvaluator(predictionCol=classifier.getPredictionCol(), + labelCol=classifier.getLabelCol(), + metricName='accuracy') + +accuracy = eval.evaluate(prediction_df) +print("\nAccuracy:", accuracy) +``` + +As an example of using the flexibility provided by DataFrames, the code below computes and displays the confusion matrix. + +```python, confusion_mtrx, results='raw' +prediction_df.groupBy(classifier.getPredictionCol()) \ + .pivot(classifier.getLabelCol()) \ + .count() \ + .sort(classifier.getPredictionCol()).show(truncate=False) +``` + +## Visualize Prediction + +Because the pipeline included a `TileExploder`, we will recreate the tiled data structure. The explosion transformation includes metadata enabling us to recreate the _tiles_. See the @ref:[`rf_assemble_tile`](reference.md#rf-assemble-tile) function documentation for more details. In this case, the pipeline is scoring on all areas, regardless of whether they intersect the label polygons. This is simply done by removing the `label` column, as @ref:[discussed above](supervised-learning.md#create-ml-pipeline). + +```python, assemble_prediction +scored = model.transform(df_mask.drop('label')) +scored.createOrReplaceTempView('scored') + +retiled = spark.sql(""" +SELECT extent, crs, + rf_assemble_tile(column_index, row_index, prediction, 128, 128) as prediction, + rf_assemble_tile(column_index, row_index, B04, 128, 128) as red, + rf_assemble_tile(column_index, row_index, B03, 128, 128) as grn, + rf_assemble_tile(column_index, row_index, B02, 128, 128) as blu +FROM scored +GROUP BY extent, crs +""") + +retiled.printSchema() +``` + +Take a look at a sample of the resulting output and the corresponding area's red-green-blue composite image. + +```python, display_rgb +sample = retiled.select('prediction', 'red', 'grn', 'blu') \ + .sort(-rf_tile_sum(rf_local_equal('prediction', lit(3.0)))) \ + .first() + +sample_prediction = sample['prediction'] + +red = sample['red'].cells +grn = sample['grn'].cells +blu = sample['blu'].cells +sample_rgb = np.concatenate([red[ :, :, None], grn[:, :, None] , blu[ :, :, None]], axis=2) +mins = np.nanmin(sample_rgb, axis=(0,1)) +plt.imshow((sample_rgb - mins)/ (np.nanmax(sample_rgb, axis=(0,1)) - mins)) +``` + +Recall the label coding: 1 is forest (purple), 2 is cropland (green) and 3 is developed areas(yellow). + +```python, display_prediction +display(sample_prediction) +``` diff --git a/pyrasterframes/src/main/python/docs/time-series.pymd b/pyrasterframes/src/main/python/docs/time-series.pymd new file mode 100644 index 000000000..4c6e7cfc2 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/time-series.pymd @@ -0,0 +1,117 @@ +# Time Series + +## Analysis Plan + +```python setup, echo=False +from IPython.display import display +import pyrasterframes +from pyrasterframes.rasterfunctions import * +import pyrasterframes.rf_ipython +# This job is more memory bound, so reduce the concurrent tasks. +spark = pyrasterframes.get_spark_session("local[4]") +``` + +In this example, we will show how the flexibility of the DataFrame concept for raster data allows a simple and intuitive way to extract a time series from Earth observation data. We will start with our @ref:[built-in MODIS data catalog](raster-catalogs.md#using-built-in-experimental-catalogs). + +```python catalog +cat = spark.read.format('aws-pds-modis-catalog').load().repartition(200) +cat.printSchema() +``` + +We will summarize the change in NDVI over 2018 in the Cuyahoga Valley National Park in Ohio, USA. First, we will retrieve open vector data delineating the park boundary from the US National Park Service's LandsNet. + +```python get_park_boundary +import requests +import geopandas +nps_data_query_url = 'https://services1.arcgis.com/fBc8EJBxQRMcHlei/arcgis/rest/services/NPS_Park_Boundaries/FeatureServer/0/query?where=1%3D1&outFields=*&geometry=-82.451%2C41.075%2C-80.682%2C41.436&geometryType=esriGeometryEnvelope&inSR=4326&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json' +r = requests.get(nps_data_query_url) +with open('/tmp/parks.geojson', 'wb') as f: + for chunk in r.iter_content(chunk_size=128): + f.write(chunk) + +park_df = geopandas.read_file('/tmp/parks.geojson') +park_geo = park_df[park_df.UNIT_CODE=='CUVA'].geometry[0] + +park_geo.wkt[:100] +``` + + +The entire park boundary is contained in MODIS granule h11 v4. We will simply filter on this granule, rather than using a @ref:[spatial relation](vector-data.md#geomesa-functions-and-spatial-relations). The time period selected should show the change in plant vigor as leaves emerge over the spring and into early summer. + +```python query_catalog +from pyspark.sql.functions import lit +park_cat = cat.filter( + (cat.granule_id == 'h11v04') & + (cat.acquisition_date > lit('2018-02-19')) & + (cat.acquisition_date < lit('2018-07-01')) + ) +park_cat.printSchema() +``` + +## Catalog Read + +Now we have a catalog with several months of MODIS data for a single granule. However, the granule is larger than our park boundary. We will combine the park geometry with the catalog, and read only the bands of interest to compute NDVI, which we discussed in a @ref:[previous section](local-algebra.md#computing-ndvi). + +```python read_catalog +raster_cols = ['B01', 'B02',] # red and near-infrared respectively +park_rf = spark.read.raster( + catalog=park_cat.select(['acquisition_date', 'granule_id'] + raster_cols), + catalog_col_names=raster_cols + ) \ + .withColumn('park', st_geomFromWKT(lit(park_geo.wkt))) +park_rf.printSchema() +park_rf.persist() +``` + +## Vector and Raster Data Interaction + +Now we have the vector representation of the park boundary alongside the _tiles_ of red and near infrared bands. Next, we need to create a _tile_ representation of the park to allow us to limit the time series analysis to pixels within the park. This is similar to the masking operation demonstrated in @ref:[NoData handling](nodata-handling.md#masking). + +We do this using two transformations. The first one will reproject the park boundary from coordinates to the MODIS sinusoidal projection. The second one will create a new _tile_ aligned with the imagery containing a value of 1 where the pixels are contained within the park and NoData elsewhere. + +```python burn_in +cr_1 = park_rf.withColumn('park_native', st_reproject('park', lit('EPSG:4326'), rf_crs('B01'))) + +cr_2 = cr_1 \ + .withColumn('dims', rf_dimensions('B01')) \ + .withColumn('park_tile', rf_rasterize('park_native', rf_geometry('B01'), lit(1), 'dims.cols', 'dims.rows')) \ + .where(rf_tile_sum('park_tile') > 0) +cr_2.printSchema() +cr_2.persist() +``` + +## Create Time Series + +Next, we will compute NDVI as the normalized difference of near infrared (band 2) and red (band 1). The _tiles_ are masked by the `park_tile`. We will then aggregate across the remaining values to arrive at an average NDVI for each week of the year. Note that the computation is creating a weighted average, which is weighted by the number of valid observations per week. + +```python ndvi_time_series +from pyspark.sql.functions import col, year, weekofyear, month +from pyspark.sql.functions import sum as sql_sum + +rf_ndvi = cr_2.withColumn('ndvi', rf_normalized_difference('B02', 'B01')) \ + .withColumn('ndvi_masked', rf_mask('ndvi', 'park_tile')) + +time_series = rf_ndvi \ + .withColumn('ndvi_wt', rf_tile_sum('ndvi_masked')) \ + .withColumn('wt', rf_data_cells('ndvi_masked')) \ + .groupby(year('acquisition_date').alias('year'), weekofyear('acquisition_date').alias('week')) \ + .agg(sql_sum('ndvi_wt').alias('ndvi_wt_wk'), sql_sum('wt').alias('wt_wk')) \ + .withColumn('ndvi', col('ndvi_wt_wk') / col('wt_wk')) +time_series.printSchema() +time_series.persist() +``` + +Finally, we will take a look at the NDVI over time. + +```python time_series_display +import matplotlib.pyplot as plt + +time_series_pdf = time_series.toPandas() +time_series_pdf = time_series_pdf.sort_values('week') +plt.plot(time_series_pdf['week'], time_series_pdf['ndvi'], 'go-') +plt.xlabel('Week of year, 2018') +plt.ylabel('NDVI') +plt.title('Cuyahoga Valley NP Green-up') +``` + +We can see two fairly clear elbows in the curve at week 17 and week 21, indicating the start and end of the green up period. Estimation of such parameters is one technique [phenology](https://en.wikipedia.org/wiki/Phenology) researchers use to monitor changes in climate and environment. diff --git a/pyrasterframes/src/main/python/docs/unsupervised-learning.pymd b/pyrasterframes/src/main/python/docs/unsupervised-learning.pymd new file mode 100644 index 000000000..c93492fa3 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/unsupervised-learning.pymd @@ -0,0 +1,136 @@ +# Unsupervised Machine Learning + +In this example, we will demonstrate how to fit and score an unsupervised learning model with a [sample of Landsat 8 data](https://github.com/locationtech/rasterframes/tree/develop/pyrasterframes/src/test/resources). + +## Imports and Data Preparation + +```python, setup, echo=False +from IPython.core.display import display +from docs import resource_dir_uri +import pyrasterframes.rf_ipython +from pyrasterframes.utils import create_rf_spark_session + +import os + +spark = create_rf_spark_session() + +``` + +We import various Spark components needed to construct our `Pipeline`. + +```python, imports, echo=True +import pandas as pd +from pyrasterframes import TileExploder +from pyrasterframes.rasterfunctions import rf_assemble_tile, rf_crs, rf_extent, rf_tile, rf_dimensions + +from pyspark.ml.feature import VectorAssembler +from pyspark.ml.clustering import KMeans +from pyspark.ml import Pipeline + +``` + +The first step is to create a Spark DataFrame of our imagery data. To achieve that we will create a catalog DataFrame using the pattern from [the I/O page](raster-io.html#Single-Scene--Multiple-Bands). In the catalog, each row represents a distinct area and time, and each column is the URI to a band's image product. The function `resource_dir_uri` gives a local file system path to the sample Landsat data. The resulting Spark DataFrame may have many rows per URI, with a column corresponding to each band. + + +```python, catalog +filenamePattern = "L8-B{}-Elkton-VA.tiff" +catalog_df = pd.DataFrame([ + {'b' + str(b): os.path.join(resource_dir_uri(), filenamePattern.format(b)) for b in range(1, 8)} +]) +df = spark.read.raster(catalog=catalog_df, catalog_col_names=catalog_df.columns) +df = df.select( + rf_crs(df.b1).alias('crs'), + rf_extent(df.b1).alias('extent'), + rf_tile(df.b1).alias('b1'), + rf_tile(df.b2).alias('b2'), + rf_tile(df.b3).alias('b3'), + rf_tile(df.b4).alias('b4'), + rf_tile(df.b5).alias('b5'), + rf_tile(df.b6).alias('b6'), + rf_tile(df.b7).alias('b7'), +) +df.printSchema() +``` + +## Create ML Pipeline + +SparkML requires that each observation be in its own row, and features for each observation be packed into a single `Vector`. For this unsupervised learning problem, we will treat each _pixel_ as an observation and each band as a feature. The first step is to "explode" the _tiles_ into a single row per pixel. In RasterFrames, generally a pixel is called a @ref:[`cell`](concepts.md#cell). + +```python, exploder +exploder = TileExploder() +``` + +To "vectorize" the the band columns, we use the SparkML `VectorAssembler`. Each of the seven bands is a different feature. + +```python, assembler +assembler = VectorAssembler() \ + .setInputCols(list(catalog_df.columns)) \ + .setOutputCol("features") +``` + +For this problem, we will use the K-means clustering algorithm and configure our model to have 5 clusters. + +```python, kmeans +kmeans = KMeans().setK(5).setFeaturesCol('features') +``` + +We can combine the above stages into a single _pipeline_. + +```python, pipeline +pipeline = Pipeline().setStages([exploder, assembler, kmeans]) +``` + +## Fit the Model and Score + +Fitting the _pipeline_ actually executes exploding the _tiles_, assembling the features _vectors_, and fitting the K-means clustering model. + +```python, fit +model = pipeline.fit(df) +``` + +We can use the `transform` function to score the training data in the fitted _pipeline_ model. This will add a column called `prediction` with the closest cluster identifier. + +```python, transform, results='raw' +clustered = model.transform(df) +clustered.show(8) +``` + +If we want to inspect the model statistics, the SparkML API requires us to go through this unfortunate contortion to access the clustering results: + +```python, cluster_stats +cluster_stage = model.stages[2] +``` + +We can then compute the sum of squared distances of points to their nearest center, which is elemental to most cluster quality metrics. + +```python, distance +metric = cluster_stage.computeCost(clustered) +print("Within set sum of squared errors: %s" % metric) +``` + +## Visualize Prediction + +We can recreate the tiled data structure using the metadata added by the `TileExploder` pipeline stage. + +```python, assemble +from pyrasterframes.rf_types import CellType + +tile_dims = df.select(rf_dimensions(df.b1).alias('dims')).first()['dims'] +retiled = clustered.groupBy('extent', 'crs') \ + .agg( + rf_assemble_tile('column_index', 'row_index', 'prediction', + tile_dims['cols'], tile_dims['rows'], CellType.int8()).alias('prediction') +) + +retiled.printSchema() +``` + +```python, results='raw' +retiled.show() +``` + +The resulting output is shown below. + +```python, viz +display(retiled.select('prediction').first()['prediction']) +``` diff --git a/pyrasterframes/src/main/python/docs/vector-data.pymd b/pyrasterframes/src/main/python/docs/vector-data.pymd new file mode 100644 index 000000000..1ec42d522 --- /dev/null +++ b/pyrasterframes/src/main/python/docs/vector-data.pymd @@ -0,0 +1,113 @@ +# Vector Data + +RasterFrames provides a variety of ways to work with spatial vector data (points, lines, and polygons) alongside raster data. There is a convenience DataSource for the GeoJSON format, as well as the ability to convert from [GeoPandas][GeoPandas] to Spark. Representation of vector geometries in PySpark is through [Shapely][Shapely], providing a great deal of interoperability. RasterFrames also provides access to Spark functions for working with geometries. + +## GeoJSON DataSource + +```python, setup, echo=False +import pyrasterframes +import pyrasterframes.rf_ipython +from pyrasterframes.utils import create_rf_spark_session +spark = create_rf_spark_session() +``` + +```python, read_geojson +from pyspark import SparkFiles +spark.sparkContext.addFile('https://raw.githubusercontent.com/datasets/geo-admin1-us/master/data/admin1-us.geojson') + +df = spark.read.geojson(SparkFiles.get('admin1-us.geojson')) +df.printSchema() +``` + +The properties of each discrete geometry are available as columns of the DataFrame, along with the geometry itself. + +## GeoPandas and RasterFrames + +You can also convert a [GeoPandas][GeoPandas] GeoDataFrame to a Spark DataFrame, preserving the geometry column. This means that any vector format that can be read with [OGR][OGR] can be converted to a Spark DataFrame. In the example below, we expect the same schema as the DataFrame defined above by the GeoJSON reader. Note that in a GeoPandas DataFrame there can be heterogeneous geometry types in the column, which may fail Spark's schema inference. + +```python, read_and_normalize +import geopandas +from shapely.geometry import MultiPolygon + +def poly_or_mp_to_mp(g): + """ Normalize polygons or multipolygons to all be multipolygons. """ + if isinstance(g, MultiPolygon): + return g + else: + return MultiPolygon([g]) + +gdf = geopandas.read_file('https://raw.githubusercontent.com/datasets/geo-admin1-us/master/data/admin1-us.geojson') +gdf.geometry = gdf.geometry.apply(poly_or_mp_to_mp) +df2 = spark.createDataFrame(gdf) +df2.printSchema() +``` + +## Shapely Geometry Support + +The `geometry` column will have a Spark user-defined type that is compatible with [Shapely][Shapely] when working with Python via PySpark. This means that when the data is collected to the driver, it will be a Shapely geometry object. + +```python, show_geom +the_first = df.first() +print(type(the_first['geometry'])) +``` + +Since it is a geometry we can do things like this: + +```python, show_wkt +the_first['geometry'].wkt +``` + +You can also write user-defined functions that take geometries as input, output, or both, via user defined types in the [geomesa_pyspark.types](https://github.com/locationtech/rasterframes/blob/develop/pyrasterframes/src/main/python/geomesa_pyspark/types.py) module. Here is a simple example of a user-defined function that uses both a geometry input and output to compute the centroid of a geometry. + +```python, add_centroid +from pyspark.sql.functions import udf +from geomesa_pyspark.types import PointUDT + +@udf(PointUDT()) +def get_centroid(g): + return g.centroid + +df = df.withColumn('naive_centroid', get_centroid(df.geometry)) +df.printSchema() +``` + +We can take a look at a sample of the data. Notice the geometry columns print as well known text (wkt). + +```python, show_centroid, results='raw' +df.show(4) +``` + + +## GeoMesa Functions and Spatial Relations + +As documented in the @ref:[function reference](reference.md), various user-defined functions implemented by GeoMesa are also available for use. The example below uses a GeoMesa user-defined function to compute the centroid of a geometry. It is logically equivalent to the example above, but more efficient. + + +```python, native_centroid, results='raw' +from pyrasterframes.rasterfunctions import st_centroid +df = df.withColumn('centroid', st_centroid(df.geometry)) +df.select('name', 'geometry', 'naive_centroid', 'centroid').show(4) +``` + +The RasterFrames vector functions and GeoMesa functions also provide a variety of spatial relations that are useful in combination with the geometric properties of projected rasters. In this example, we use the @ref:[built-in Landsat catalog](raster-catalogs.md#using-built-in-experimental-catalogs) which provides an extent. We will convert the extent to a polygon and filter to those within approximately 500 km of a selected point. + +```python, evaluate=True +from pyrasterframes.rasterfunctions import st_geometry, st_bufferPoint, st_intersects, st_point +from pyspark.sql.functions import lit +l8 = spark.read.format('aws-pds-l8-catalog').load() + +l8 = l8.withColumn('geom', st_geometry(l8.bounds_wgs84)) +l8 = l8.withColumn('paducah', st_point(lit(-88.6275), lit(37.072222))) + +l8_filtered = l8.filter(st_intersects(l8.geom, st_bufferPoint(l8.paducah, lit(500000.0)))) +``` + +```python, evaluate=False, echo=False +# suppressed due to run time. +l8_filtered.count() +``` + + +[GeoPandas]: http://geopandas.org +[OGR]: https://gdal.org/drivers/vector/index.html +[Shapely]: https://shapely.readthedocs.io/en/latest/manual.html diff --git a/pyrasterframes/python/geomesa_pyspark/__init__.py b/pyrasterframes/src/main/python/geomesa_pyspark/__init__.py similarity index 100% rename from pyrasterframes/python/geomesa_pyspark/__init__.py rename to pyrasterframes/src/main/python/geomesa_pyspark/__init__.py diff --git a/pyrasterframes/src/main/python/geomesa_pyspark/types.py b/pyrasterframes/src/main/python/geomesa_pyspark/types.py new file mode 100644 index 000000000..5f1d0a110 --- /dev/null +++ b/pyrasterframes/src/main/python/geomesa_pyspark/types.py @@ -0,0 +1,94 @@ +"""*********************************************************************** + This file was created by Astraea, Inc., 2018 from an excerpt of the + original: + + Copyright (c) 2013-2018 Commonwealth Computer Research, Inc. + All rights reserved. This program and the accompanying materials + are made available under the terms of the Apache License, Version 2.0 + which accompanies this distribution and is available at + http://www.opensource.org/licenses/apache2.0.php. ++ ***********************************************************************/""" + +from pyspark.sql.types import UserDefinedType, StructField, BinaryType, StructType +from shapely import wkb +from shapely.geometry import LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon +from shapely.geometry.base import BaseGeometry +from shapely.geometry.collection import GeometryCollection + + +class ShapelyGeometryUDT(UserDefinedType): + + @classmethod + def sqlType(cls): + return StructType([StructField("wkb", BinaryType(), True)]) + + @classmethod + def module(cls): + return 'geomesa_pyspark.types' + + @classmethod + def scalaUDT(cls): + return 'org.apache.spark.sql.jts.' + cls.__name__ + + def serialize(self, obj): + return [_serialize_to_wkb(obj)] + + def deserialize(self, datum): + return _deserialize_from_wkb(datum[0]) + + +class PointUDT(ShapelyGeometryUDT): + pass + + +class LineStringUDT(ShapelyGeometryUDT): + pass + + +class PolygonUDT(ShapelyGeometryUDT): + pass + + +class MultiPointUDT(ShapelyGeometryUDT): + pass + + +class MultiLineStringUDT(ShapelyGeometryUDT): + pass + + +class MultiPolygonUDT(ShapelyGeometryUDT): + pass + + +class GeometryUDT(ShapelyGeometryUDT): + pass + + +class GeometryCollectionUDT(ShapelyGeometryUDT): + pass + + +def _serialize_to_wkb(data): + if isinstance(data, BaseGeometry): + return bytearray(data.wkb) # bytearray(...) needed for Python 2 compat. + return None + + +def _deserialize_from_wkb(data): + if data is None: + return None + return wkb.loads(bytes(data)) # bytes(...) needed for Python 2 compat. + + +_deserialize_from_wkb.__safe_for_unpickling__ = True + +# inject some PySpark constructs into Shapely's geometry types +Point.__UDT__ = PointUDT() +MultiPoint.__UDT__ = MultiPointUDT() +LineString.__UDT__ = LineStringUDT() +MultiLineString.__UDT__ = MultiLineStringUDT() +Polygon.__UDT__ = PolygonUDT() +MultiPolygon.__UDT__ = MultiPolygonUDT() +BaseGeometry.__UDT__ = GeometryUDT() +GeometryCollection.__UDT__ = GeometryCollectionUDT() diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py new file mode 100644 index 000000000..1fa5e91cf --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -0,0 +1,227 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Module initialization for PyRasterFrames. This is where much of the cool stuff is +appended to PySpark classes. +""" + +from __future__ import absolute_import +from pyspark import SparkContext +from pyspark.sql import SparkSession, DataFrame, DataFrameReader, DataFrameWriter +from pyspark.sql.column import _to_java_column + +# Import RasterFrameLayer types and functions +from .rf_context import RFContext +from .version import __version__ +from .rf_types import RasterFrameLayer, TileExploder, TileUDT, RasterSourceUDT + +__all__ = ['RasterFrameLayer', 'TileExploder'] + + +def _rf_init(spark_session): + """ Adds RasterFrames functionality to PySpark session.""" + if not hasattr(spark_session, "rasterframes"): + spark_session.rasterframes = RFContext(spark_session) + spark_session.sparkContext._rf_context = spark_session.rasterframes + + return spark_session + + +def _kryo_init(builder): + """Registers Kryo Serializers for better performance.""" + # NB: These methods need to be kept up-to-date wit those in `org.locationtech.rasterframes.extensions.KryoMethods` + builder \ + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") \ + .config("spark.kryo.registrator", "org.locationtech.rasterframes.util.RFKryoRegistrator") \ + .config("spark.kryoserializer.buffer.max", "500m") + return builder + + +def get_spark_session(master="local[*]"): + """ Create a SparkSession with pyrasterframes enabled and configured. """ + from pyrasterframes.utils import create_rf_spark_session + + return create_rf_spark_session(master) + + +def _convert_df(df, sp_key=None, metadata=None): + ctx = SparkContext._active_spark_context._rf_context + + if sp_key is None: + return RasterFrameLayer(ctx._jrfctx.asLayer(df._jdf), ctx._spark_session) + else: + import json + return RasterFrameLayer(ctx._jrfctx.asLayer( + df._jdf, _to_java_column(sp_key), json.dumps(metadata)), ctx._spark_session) + + +def _raster_join(df, other, left_extent=None, left_crs=None, right_extent=None, right_crs=None, join_exprs=None): + ctx = SparkContext._active_spark_context._rf_context + if join_exprs is not None: + assert left_extent is not None and left_crs is not None and right_extent is not None and right_crs is not None + # Note the order of arguments here. + cols = [join_exprs, left_extent, left_crs, right_extent, right_crs] + jdf = ctx._jrfctx.rasterJoin(df._jdf, other._jdf, *[_to_java_column(c) for c in cols]) + + elif left_extent is not None: + assert left_crs is not None and right_extent is not None and right_crs is not None + cols = [left_extent, left_crs, right_extent, right_crs] + jdf = ctx._jrfctx.rasterJoin(df._jdf, other._jdf, *[_to_java_column(c) for c in cols]) + + else: + jdf = ctx._jrfctx.rasterJoin(df._jdf, other._jdf) + + return RasterFrameLayer(jdf, ctx._spark_session) + + +def _layer_reader(df_reader, format_key, path, **options): + """ Loads the file of the given type at the given path.""" + df = df_reader.format(format_key).load(path, **options) + return _convert_df(df) + + +def _aliased_reader(df_reader, format_key, path, **options): + """ Loads the file of the given type at the given path.""" + return df_reader.format(format_key).load(path, **options) + + +def _aliased_writer(df_writer, format_key, path, **options): + """ Saves the dataframe to a file of the given type at the given path.""" + return df_writer.format(format_key).save(path, **options) + + +def _raster_reader( + df_reader, + path=None, + catalog=None, + catalog_col_names=None, + band_indexes=None, + tile_dimensions=(256, 256), + lazy_tiles=True, + **options): + + from pandas import DataFrame as PdDataFrame + + def to_csv(comp): + if isinstance(comp, str): + return comp + else: + return ','.join(str(v) for v in comp) + + def temp_name(): + """ Create a random name for a temporary view """ + import uuid + return str(uuid.uuid4()).replace('-', '') + + if band_indexes is None: + band_indexes = [0] + + options.update({ + "bandIndexes": to_csv(band_indexes), + "tileDimensions": to_csv(tile_dimensions), + "lazyTiles": lazy_tiles + }) + + if catalog is not None: + if catalog_col_names is None: + raise Exception("'catalog_col_names' required when DataFrame 'catalog' specified") + if isinstance(catalog, str): + options.update({ + "catalogCSV": catalog, + "catalogColumns": to_csv(catalog_col_names) + }) + elif isinstance(catalog, DataFrame): + # Create a random view name + tmp_name = temp_name() + catalog.createOrReplaceTempView(tmp_name) + options.update({ + "catalogTable": tmp_name, + "catalogColumns": to_csv(catalog_col_names) + }) + elif isinstance(catalog, PdDataFrame): + # Handle to active spark session + session = SparkContext._active_spark_context._rf_context._spark_session + # Create a random view name + tmp_name = temp_name() + spark_catalog = session.createDataFrame(catalog) + spark_catalog.createOrReplaceTempView(tmp_name) + options.update({ + "catalogTable": tmp_name, + "catalogColumns": to_csv(catalog_col_names) + }) + + return df_reader \ + .format("raster") \ + .load(path, **options) + + +def _geotiff_writer( + df_writer, + path=None, + crs=None, + raster_dimensions=None, + **options): + + def set_dims(parts): + parts = [int(p) for p in parts] + assert len(parts) == 2, "Expected dimensions specification to have exactly two components" + assert all([p > 0 for p in parts]), "Expected all components in dimensions to be positive integers" + options.update({ + "imageWidth": parts[0], + "imageHeight": parts[1] + }) + parts = [int(p) for p in parts] + assert all([p > 0 for p in parts]), 'nice message' + + if raster_dimensions is not None: + if isinstance(raster_dimensions, (list, tuple)): + set_dims(raster_dimensions) + elif isinstance(raster_dimensions, str): + set_dims(raster_dimensions.split(',')) + + if crs is not None: + options.update({ + "crs": crs + }) + + return _aliased_writer(df_writer, "geotiff", path, **options) + + +# Patch RasterFrames initialization method on SparkSession to mirror Scala approach +SparkSession.withRasterFrames = _rf_init + +# Patch Kryo serialization initialization method on SparkSession.Builder to mirror Scala approach +SparkSession.Builder.withKryoSerialization = _kryo_init + +# Add the 'asLayer' method to pyspark DataFrame +DataFrame.as_layer = _convert_df + +# Add `raster_join` method to pyspark DataFrame +DataFrame.raster_join = _raster_join + +# Add DataSource convenience methods to the DataFrameReader +DataFrameReader.raster = _raster_reader +DataFrameReader.geojson = lambda df_reader, path: _aliased_reader(df_reader, "geojson", path) +DataFrameReader.geotiff = lambda df_reader, path: _layer_reader(df_reader, "geotiff", path) +DataFrameWriter.geotiff = _geotiff_writer +DataFrameReader.geotrellis = lambda df_reader, path: _layer_reader(df_reader, "geotrellis", path) +DataFrameReader.geotrellis_catalog = lambda df_reader, path: _aliased_reader(df_reader, "geotrellis-catalog", path) +DataFrameWriter.geotrellis = lambda df_writer, path: _aliased_writer(df_writer, "geotrellis", path) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py new file mode 100644 index 000000000..6545b5a72 --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -0,0 +1,896 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +This module creates explicit Python functions that map back to the existing Scala +implementations. Most functions are standard Column functions, but those with unique +signatures are handled here as well. +""" +from __future__ import absolute_import +from pyspark.sql.column import Column, _to_java_column +from .rf_context import RFContext +from .rf_types import CellType + +THIS_MODULE = 'pyrasterframes' + + +def _context_call(name, *args): + f = RFContext.active().lookup(name) + return f(*args) + + +def _parse_cell_type(cell_type_arg): + """ Convert the cell type representation to the expected JVM CellType object.""" + + def to_jvm(ct): + return _context_call('_parse_cell_type', ct) + + if isinstance(cell_type_arg, str): + return to_jvm(cell_type_arg) + elif isinstance(cell_type_arg, CellType): + return to_jvm(cell_type_arg.cell_type_name) + + +def rf_cell_types(): + """Return a list of standard cell types""" + return [CellType(str(ct)) for ct in _context_call('rf_cell_types')] + + +def rf_assemble_tile(col_index, row_index, cell_data_col, num_cols, num_rows, cell_type): + """Create a Tile from a column of cell data with location indices""" + jfcn = RFContext.active().lookup('rf_assemble_tile') + return Column( + jfcn(_to_java_column(col_index), _to_java_column(row_index), _to_java_column(cell_data_col), num_cols, num_rows, + _parse_cell_type(cell_type))) + + +def rf_array_to_tile(array_col, num_cols, num_rows): + """Convert array in `array_col` into a Tile of dimensions `num_cols` and `num_rows'""" + jfcn = RFContext.active().lookup('rf_array_to_tile') + return Column(jfcn(_to_java_column(array_col), num_cols, num_rows)) + + +def rf_convert_cell_type(tile_col, cell_type): + """Convert the numeric type of the Tiles in `tileCol`""" + jfcn = RFContext.active().lookup('rf_convert_cell_type') + return Column(jfcn(_to_java_column(tile_col), _parse_cell_type(cell_type))) + + +def rf_make_constant_tile(scalar_value, num_cols, num_rows, cell_type=CellType.float64()): + """Constructor for constant tile column""" + jfcn = RFContext.active().lookup('rf_make_constant_tile') + return Column(jfcn(scalar_value, num_cols, num_rows, _parse_cell_type(cell_type))) + + +def rf_make_zeros_tile(num_cols, num_rows, cell_type=CellType.float64()): + """Create column of constant tiles of zero""" + jfcn = RFContext.active().lookup('rf_make_zeros_tile') + return Column(jfcn(num_cols, num_rows, _parse_cell_type(cell_type))) + + +def rf_make_ones_tile(num_cols, num_rows, cell_type=CellType.float64()): + """Create column of constant tiles of one""" + jfcn = RFContext.active().lookup('rf_make_ones_tile') + return Column(jfcn(num_cols, num_rows, _parse_cell_type(cell_type))) + + +def rf_rasterize(geometry_col, bounds_col, value_col, num_cols_col, num_rows_col): + """Create a tile where cells in the grid defined by cols, rows, and bounds are filled with the given value.""" + jfcn = RFContext.active().lookup('rf_rasterize') + return Column(jfcn(_to_java_column(geometry_col), _to_java_column(bounds_col), + _to_java_column(value_col), _to_java_column(num_cols_col), _to_java_column(num_rows_col))) + + +def st_reproject(geometry_col, src_crs, dst_crs): + """Reproject a column of geometry given the CRSs of the source and destination.""" + jfcn = RFContext.active().lookup('st_reproject') + return Column(jfcn(_to_java_column(geometry_col), _to_java_column(src_crs), _to_java_column(dst_crs))) + + +def rf_explode_tiles(*tile_cols): + """Create a row for each cell in Tile.""" + jfcn = RFContext.active().lookup('rf_explode_tiles') + jcols = [_to_java_column(arg) for arg in tile_cols] + return Column(jfcn(RFContext.active().list_to_seq(jcols))) + + +def rf_explode_tiles_sample(sample_frac, seed, *tile_cols): + """Create a row for a sample of cells in Tile columns.""" + jfcn = RFContext.active().lookup('rf_explode_tiles_sample') + jcols = [_to_java_column(arg) for arg in tile_cols] + return Column(jfcn(sample_frac, seed, RFContext.active().list_to_seq(jcols))) + + +def rf_mask_by_value(data_tile, mask_tile, mask_value): + """Generate a tile with the values from the data tile, but where cells in the masking tile contain the masking + value, replace the data value with NODATA. """ + jfcn = RFContext.active().lookup('rf_mask_by_value') + return Column(jfcn(_to_java_column(data_tile), _to_java_column(mask_tile), _to_java_column(mask_value))) + + +def rf_inverse_mask_by_value(data_tile, mask_tile, mask_value): + """Generate a tile with the values from the data tile, but where cells in the masking tile do not contain the + masking value, replace the data value with NODATA. """ + jfcn = RFContext.active().lookup('rf_inverse_mask_by_value') + return Column(jfcn(_to_java_column(data_tile), _to_java_column(mask_tile), _to_java_column(mask_value))) + + +def _apply_scalar_to_tile(name, tile_col, scalar): + jfcn = RFContext.active().lookup(name) + return Column(jfcn(_to_java_column(tile_col), scalar)) + + +def rf_with_no_data(tile_col, scalar): + """Assign a `NoData` value to the Tiles in the given Column.""" + return _apply_scalar_to_tile('rf_with_no_data', tile_col, scalar) + + +def rf_local_add_double(tile_col, scalar): + """Add a floating point scalar to a Tile""" + return _apply_scalar_to_tile('rf_local_add_double', tile_col, scalar) + + +def rf_local_add_int(tile_col, scalar): + """Add an integral scalar to a Tile""" + return _apply_scalar_to_tile('rf_local_add_int', tile_col, scalar) + + +def rf_local_subtract_double(tile_col, scalar): + """Subtract a floating point scalar from a Tile""" + return _apply_scalar_to_tile('rf_local_subtract_double', tile_col, scalar) + + +def rf_local_subtract_int(tile_col, scalar): + """Subtract an integral scalar from a Tile""" + return _apply_scalar_to_tile('rf_local_subtract_int', tile_col, scalar) + + +def rf_local_multiply_double(tile_col, scalar): + """Multiply a Tile by a float point scalar""" + return _apply_scalar_to_tile('rf_local_multiply_double', tile_col, scalar) + + +def rf_local_multiply_int(tile_col, scalar): + """Multiply a Tile by an integral scalar""" + return _apply_scalar_to_tile('rf_local_multiply_int', tile_col, scalar) + + +def rf_local_divide_double(tile_col, scalar): + """Divide a Tile by a floating point scalar""" + return _apply_scalar_to_tile('rf_local_divide_double', tile_col, scalar) + + +def rf_local_divide_int(tile_col, scalar): + """Divide a Tile by an integral scalar""" + return _apply_scalar_to_tile('rf_local_divide_int', tile_col, scalar) + + +def rf_local_less_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is less than a scalar, otherwise 0""" + return _apply_scalar_to_tile('foo', tile_col, scalar) + + +def rf_local_less_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is less than a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_less_double', tile_col, scalar) + + +def rf_local_less_equal_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is less than or equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_less_equal_double', tile_col, scalar) + + +def rf_local_less_equal_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is less than or equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_less_equal_int', tile_col, scalar) + + +def rf_local_greater_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is greater than a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_greater_double', tile_col, scalar) + + +def rf_local_greater_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is greater than a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_greater_int', tile_col, scalar) + + +def rf_local_greater_equal_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is greater than or equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_greater_equal_double', tile_col, scalar) + + +def rf_local_greater_equal_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is greater than or equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_greater_equal_int', tile_col, scalar) + + +def rf_local_equal_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_equal_double', tile_col, scalar) + + +def rf_local_equal_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_equal_int', tile_col, scalar) + + +def rf_local_unequal_double(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_unequal_double', tile_col, scalar) + + +def rf_local_unequal_int(tile_col, scalar): + """Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0""" + return _apply_scalar_to_tile('rf_local_unequal_int', tile_col, scalar) + + +def _apply_column_function(name, *args): + jfcn = RFContext.active().lookup(name) + jcols = [_to_java_column(arg) for arg in args] + return Column(jfcn(*jcols)) + + +def rf_dimensions(tile_col): + """Query the number of (cols, rows) in a Tile.""" + return _apply_column_function('rf_dimensions', tile_col) + + +def rf_tile_to_array_int(tile_col): + """Flattens Tile into an array of integers.""" + return _apply_column_function('rf_tile_to_array_int', tile_col) + + +def rf_tile_to_array_double(tile_col): + """Flattens Tile into an array of doubles.""" + return _apply_column_function('rf_tile_to_array_double', tile_col) + + +def rf_cell_type(tile_col): + """Extract the Tile's cell type""" + return _apply_column_function('rf_cell_type', tile_col) + + +def rf_is_no_data_tile(tile_col): + """Report if the Tile is entirely NODDATA cells""" + return _apply_column_function('rf_is_no_data_tile', tile_col) + + +def rf_exists(tile_col): + """Returns true if any cells in the tile are true (non-zero and not NoData)""" + return _apply_column_function('rf_exists', tile_col) + + +def rf_for_all(tile_col): + """Returns true if all cells in the tile are true (non-zero and not NoData).""" + return _apply_column_function('rf_for_all', tile_col) + + +def rf_agg_approx_histogram(tile_col): + """Compute the full column aggregate floating point histogram""" + return _apply_column_function('rf_agg_approx_histogram', tile_col) + + +def rf_agg_stats(tile_col): + """Compute the full column aggregate floating point statistics""" + return _apply_column_function('rf_agg_stats', tile_col) + + +def rf_agg_mean(tile_col): + """Computes the column aggregate mean""" + return _apply_column_function('rf_agg_mean', tile_col) + + +def rf_agg_data_cells(tile_col): + """Computes the number of non-NoData cells in a column""" + return _apply_column_function('rf_agg_data_cells', tile_col) + + +def rf_agg_no_data_cells(tile_col): + """Computes the number of NoData cells in a column""" + return _apply_column_function('rf_agg_no_data_cells', tile_col) + + +def rf_tile_histogram(tile_col): + """Compute the Tile-wise histogram""" + return _apply_column_function('rf_tile_histogram', tile_col) + + +def rf_tile_mean(tile_col): + """Compute the Tile-wise mean""" + return _apply_column_function('rf_tile_mean', tile_col) + + +def rf_tile_sum(tile_col): + """Compute the Tile-wise sum""" + return _apply_column_function('rf_tile_sum', tile_col) + + +def rf_tile_min(tile_col): + """Compute the Tile-wise minimum""" + return _apply_column_function('rf_tile_min', tile_col) + + +def rf_tile_max(tile_col): + """Compute the Tile-wise maximum""" + return _apply_column_function('rf_tile_max', tile_col) + + +def rf_tile_stats(tile_col): + """Compute the Tile-wise floating point statistics""" + return _apply_column_function('rf_tile_stats', tile_col) + + +def rf_render_ascii(tile_col): + """Render ASCII art of tile""" + return _apply_column_function('rf_render_ascii', tile_col) + + +def rf_render_matrix(tile_col): + """Render Tile cell values as numeric values, for debugging purposes""" + return _apply_column_function('rf_render_matrix', tile_col) + + +def rf_no_data_cells(tile_col): + """Count of NODATA cells""" + return _apply_column_function('rf_no_data_cells', tile_col) + + +def rf_data_cells(tile_col): + """Count of cells with valid data""" + return _apply_column_function('rf_data_cells', tile_col) + + +def rf_local_add(left_tile_col, right_tile_col): + """Add two Tiles""" + return _apply_column_function('rf_local_add', left_tile_col, right_tile_col) + + +def rf_local_subtract(left_tile_col, right_tile_col): + """Subtract two Tiles""" + return _apply_column_function('rf_local_subtract', left_tile_col, right_tile_col) + + +def rf_local_multiply(left_tile_col, right_tile_col): + """Multiply two Tiles""" + return _apply_column_function('rf_local_multiply', left_tile_col, right_tile_col) + + +def rf_local_divide(left_tile_col, right_tile_col): + """Divide two Tiles""" + return _apply_column_function('rf_local_divide', left_tile_col, right_tile_col) + + +def rf_normalized_difference(left_tile_col, right_tile_col): + """Compute the normalized difference of two tiles""" + return _apply_column_function('rf_normalized_difference', left_tile_col, right_tile_col) + + +def rf_agg_local_max(tile_col): + """Compute the cell-wise/local max operation between Tiles in a column.""" + return _apply_column_function('rf_agg_local_max', tile_col) + + +def rf_agg_local_min(tile_col): + """Compute the cellwise/local min operation between Tiles in a column.""" + return _apply_column_function('rf_agg_local_min', tile_col) + + +def rf_agg_local_mean(tile_col): + """Compute the cellwise/local mean operation between Tiles in a column.""" + return _apply_column_function('rf_agg_local_mean', tile_col) + + +def rf_agg_local_data_cells(tile_col): + """Compute the cellwise/local count of non-NoData cells for all Tiles in a column.""" + return _apply_column_function('rf_agg_local_data_cells', tile_col) + + +def rf_agg_local_no_data_cells(tile_col): + """Compute the cellwise/local count of NoData cells for all Tiles in a column.""" + return _apply_column_function('rf_agg_local_no_data_cells', tile_col) + + +def rf_agg_local_stats(tile_col): + """Compute cell-local aggregate descriptive statistics for a column of Tiles.""" + return _apply_column_function('rf_agg_local_stats', tile_col) + + +def rf_mask(src_tile_col, mask_tile_col): + """Where the rf_mask (second) tile contains NODATA, replace values in the source (first) tile with NODATA.""" + return _apply_column_function('rf_mask', src_tile_col, mask_tile_col) + + +def rf_inverse_mask(src_tile_col, mask_tile_col): + """Where the rf_mask (second) tile DOES NOT contain NODATA, replace values in the source (first) tile with NODATA.""" + return _apply_column_function('rf_inverse_mask', src_tile_col, mask_tile_col) + + +def rf_local_less(left_tile_col, right_tile_col): + """Cellwise less than comparison between two tiles""" + return _apply_column_function('rf_local_less', left_tile_col, right_tile_col) + + +def rf_local_less_equal(left_tile_col, right_tile_col): + """Cellwise less than or equal to comparison between two tiles""" + return _apply_column_function('rf_local_less_equal', left_tile_col, right_tile_col) + + +def rf_local_greater(left_tile_col, right_tile_col): + """Cellwise greater than comparison between two tiles""" + return _apply_column_function('rf_local_greater', left_tile_col, right_tile_col) + + +def rf_local_greater_equal(left_tile_col, right_tile_col): + """Cellwise greater than or equal to comparison between two tiles""" + return _apply_column_function('rf_local_greater_equal', left_tile_col, right_tile_col) + + +def rf_local_equal(left_tile_col, right_tile_col): + """Cellwise equality comparison between two tiles""" + return _apply_column_function('rf_local_equal', left_tile_col, right_tile_col) + + +def rf_local_unequal(left_tile_col, right_tile_col): + """Cellwise inequality comparison between two tiles""" + return _apply_column_function('rf_local_unequal', left_tile_col, right_tile_col) + + +def rf_round(tile_col): + """Round cell values to the nearest integer without changing the cell type""" + return _apply_column_function('rf_round', tile_col) + + +def rf_abs(tile_col): + """Compute the absolute value of each cell""" + return _apply_column_function('rf_abs', tile_col) + + +def rf_log(tile_col): + """Performs cell-wise natural logarithm""" + return _apply_column_function('rf_log', tile_col) + + +def rf_log10(tile_col): + """Performs cell-wise logartithm with base 10""" + return _apply_column_function('rf_log10', tile_col) + + +def rf_log2(tile_col): + """Performs cell-wise logartithm with base 2""" + return _apply_column_function('rf_log2', tile_col) + + +def rf_log1p(tile_col): + """Performs natural logarithm of cell values plus one""" + return _apply_column_function('rf_log1p', tile_col) + + +def rf_exp(tile_col): + """Performs cell-wise exponential""" + return _apply_column_function('rf_exp', tile_col) + + +def rf_exp2(tile_col): + """Compute 2 to the power of cell values""" + return _apply_column_function('rf_exp2', tile_col) + + +def rf_exp10(tile_col): + """Compute 10 to the power of cell values""" + return _apply_column_function('rf_exp10', tile_col) + + +def rf_expm1(tile_col): + """Performs cell-wise exponential, then subtract one""" + return _apply_column_function('rf_expm1', tile_col) + + +def rf_identity(tile_col): + """Pass tile through unchanged""" + return _apply_column_function('rf_identity', tile_col) + + +def rf_resample(tile_col, scale_factor_col): + """Resample tile to different size based on scalar factor or tile whose dimension to match + Scalar less than one will downsample tile; greater than one will upsample. Uses nearest-neighbor.""" + return _apply_column_function('rf_resample', tile_col, scale_factor_col) + + +def rf_crs(tile_col): + """Get the CRS of a RasterSource or ProjectedRasterTile""" + return _apply_column_function('rf_crs', tile_col) + + +def rf_mk_crs(crs_text): + """Resolve CRS from text identifier. Supported registries are EPSG, ESRI, WORLD, NAD83, & NAD27. + An example of a valid CRS name is EPSG:3005.""" + return Column(_context_call('_make_crs_literal', crs_text)) + + +def st_extent(geom_col): + """Compute the extent/bbox of a Geometry (a tile with embedded extent and CRS)""" + return _apply_column_function('st_extent', geom_col) + + +def rf_extent(proj_raster_col): + """Get the extent of a RasterSource or ProjectedRasterTile (a tile with embedded extent and CRS)""" + return _apply_column_function('rf_extent', proj_raster_col) + + +def rf_tile(proj_raster_col): + """Extracts the Tile component of a ProjectedRasterTile (or Tile).""" + return _apply_column_function('rf_tile', proj_raster_col) + + +def st_geometry(geom_col): + """Convert the given extent/bbox to a polygon""" + return _apply_column_function('st_geometry', geom_col) + + +def rf_geometry(proj_raster_col): + """Get the extent of a RasterSource or ProjectdRasterTile as a Geometry""" + return _apply_column_function('rf_geometry', proj_raster_col) + +# ------ GeoMesa Functions ------ + +def st_geomFromGeoHash(*args): + """""" + return _apply_column_function('st_geomFromGeoHash', *args) + + +def st_geomFromWKT(*args): + """""" + return _apply_column_function('st_geomFromWKT', *args) + + +def st_geomFromWKB(*args): + """""" + return _apply_column_function('st_geomFromWKB', *args) + + +def st_lineFromText(*args): + """""" + return _apply_column_function('st_lineFromText', *args) + + +def st_makeBox2D(*args): + """""" + return _apply_column_function('st_makeBox2D', *args) + + +def st_makeBBox(*args): + """""" + return _apply_column_function('st_makeBBox', *args) + + +def st_makePolygon(*args): + """""" + return _apply_column_function('st_makePolygon', *args) + + +def st_makePoint(*args): + """""" + return _apply_column_function('st_makePoint', *args) + + +def st_makeLine(*args): + """""" + return _apply_column_function('st_makeLine', *args) + + +def st_makePointM(*args): + """""" + return _apply_column_function('st_makePointM', *args) + + +def st_mLineFromText(*args): + """""" + return _apply_column_function('st_mLineFromText', *args) + + +def st_mPointFromText(*args): + """""" + return _apply_column_function('st_mPointFromText', *args) + + +def st_mPolyFromText(*args): + """""" + return _apply_column_function('st_mPolyFromText', *args) + + +def st_point(*args): + """""" + return _apply_column_function('st_point', *args) + + +def st_pointFromGeoHash(*args): + """""" + return _apply_column_function('st_pointFromGeoHash', *args) + + +def st_pointFromText(*args): + """""" + return _apply_column_function('st_pointFromText', *args) + + +def st_pointFromWKB(*args): + """""" + return _apply_column_function('st_pointFromWKB', *args) + + +def st_polygon(*args): + """""" + return _apply_column_function('st_polygon', *args) + + +def st_polygonFromText(*args): + """""" + return _apply_column_function('st_polygonFromText', *args) + + +def st_castToPoint(*args): + """""" + return _apply_column_function('st_castToPoint', *args) + + +def st_castToPolygon(*args): + """""" + return _apply_column_function('st_castToPolygon', *args) + + +def st_castToLineString(*args): + """""" + return _apply_column_function('st_castToLineString', *args) + + +def st_byteArray(*args): + """""" + return _apply_column_function('st_byteArray', *args) + + +def st_boundary(*args): + """""" + return _apply_column_function('st_boundary', *args) + + +def st_coordDim(*args): + """""" + return _apply_column_function('st_coordDim', *args) + + +def st_dimension(*args): + """""" + return _apply_column_function('st_dimension', *args) + + +def st_envelope(*args): + """""" + return _apply_column_function('st_envelope', *args) + + +def st_exteriorRing(*args): + """""" + return _apply_column_function('st_exteriorRing', *args) + + +def st_geometryN(*args): + """""" + return _apply_column_function('st_geometryN', *args) + + +def st_geometryType(*args): + """""" + return _apply_column_function('st_geometryType', *args) + + +def st_interiorRingN(*args): + """""" + return _apply_column_function('st_interiorRingN', *args) + + +def st_isClosed(*args): + """""" + return _apply_column_function('st_isClosed', *args) + + +def st_isCollection(*args): + """""" + return _apply_column_function('st_isCollection', *args) + + +def st_isEmpty(*args): + """""" + return _apply_column_function('st_isEmpty', *args) + + +def st_isRing(*args): + """""" + return _apply_column_function('st_isRing', *args) + + +def st_isSimple(*args): + """""" + return _apply_column_function('st_isSimple', *args) + + +def st_isValid(*args): + """""" + return _apply_column_function('st_isValid', *args) + + +def st_numGeometries(*args): + """""" + return _apply_column_function('st_numGeometries', *args) + + +def st_numPoints(*args): + """""" + return _apply_column_function('st_numPoints', *args) + + +def st_pointN(*args): + """""" + return _apply_column_function('st_pointN', *args) + + +def st_x(*args): + """""" + return _apply_column_function('st_x', *args) + + +def st_y(*args): + """""" + return _apply_column_function('st_y', *args) + + +def st_asBinary(*args): + """""" + return _apply_column_function('st_asBinary', *args) + + +def st_asGeoJSON(*args): + """""" + return _apply_column_function('st_asGeoJSON', *args) + + +def st_asLatLonText(*args): + """""" + return _apply_column_function('st_asLatLonText', *args) + + +def st_asText(*args): + """""" + return _apply_column_function('st_asText', *args) + + +def st_geoHash(*args): + """""" + return _apply_column_function('st_geoHash', *args) + + +def st_bufferPoint(*args): + """""" + return _apply_column_function('st_bufferPoint', *args) + + +def st_antimeridianSafeGeom(*args): + """""" + return _apply_column_function('st_antimeridianSafeGeom', *args) + + +def st_translate(*args): + """""" + return _apply_column_function('st_translate', *args) + + +def st_contains(*args): + """""" + return _apply_column_function('st_contains', *args) + + +def st_covers(*args): + """""" + return _apply_column_function('st_covers', *args) + + +def st_crosses(*args): + """""" + return _apply_column_function('st_crosses', *args) + + +def st_disjoint(*args): + """""" + return _apply_column_function('st_disjoint', *args) + + +def st_equals(*args): + """""" + return _apply_column_function('st_equals', *args) + + +def st_intersects(*args): + """""" + return _apply_column_function('st_intersects', *args) + + +def st_overlaps(*args): + """""" + return _apply_column_function('st_overlaps', *args) + + +def st_touches(*args): + """""" + return _apply_column_function('st_touches', *args) + + +def st_within(*args): + """""" + return _apply_column_function('st_within', *args) + + +def st_relate(*args): + """""" + return _apply_column_function('st_relate', *args) + + +def st_relateBool(*args): + """""" + return _apply_column_function('st_relateBool', *args) + + +def st_area(*args): + """""" + return _apply_column_function('st_area', *args) + + +def st_closestPoint(*args): + """""" + return _apply_column_function('st_closestPoint', *args) + + +def st_centroid(*args): + """""" + return _apply_column_function('st_centroid', *args) + + +def st_distance(*args): + """""" + return _apply_column_function('st_distance', *args) + + +def st_distanceSphere(*args): + """""" + return _apply_column_function('st_distanceSphere', *args) + + +def st_length(*args): + """""" + return _apply_column_function('st_length', *args) + + +def st_aggregateDistanceSphere(*args): + """""" + return _apply_column_function('st_aggregateDistanceSphere', *args) + + +def st_lengthSphere(*args): + """""" + return _apply_column_function('st_lengthSphere', *args) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_context.py b/pyrasterframes/src/main/python/pyrasterframes/rf_context.py new file mode 100644 index 000000000..0b96a6348 --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/rf_context.py @@ -0,0 +1,85 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +This module contains access to the jvm SparkContext with RasterFrameLayer support. +""" + +from pyspark import SparkContext + +__all__ = ['RFContext'] + + +class RFContext(object): + """ + Entrypoint to RasterFrames services + """ + def __init__(self, spark_session): + self._spark_session = spark_session + self._gateway = spark_session.sparkContext._gateway + self._jvm = self._gateway.jvm + jsess = self._spark_session._jsparkSession + self._jrfctx = self._jvm.org.locationtech.rasterframes.py.PyRFContext(jsess) + + def list_to_seq(self, py_list): + conv = self.lookup('_listToSeq') + return conv(py_list) + + def lookup(self, function_name): + return getattr(self._jrfctx, function_name) + + def build_info(self): + return self._jrfctx.buildInfo() + + # NB: Tightly coupled to `org.locationtech.rasterframes.py.PyRFContext._resolveRasterRef` + def _resolve_raster_ref(self, ref): + f = self.lookup("_resolveRasterRef") + return f( + ref.source.raster_source_kryo, + ref.bandIndex, + ref.subextent.xmin, + ref.subextent.ymin, + ref.subextent.xmax, + ref.subextent.ymax, + ) + + @staticmethod + def active(): + """ + Get the active Python RFContext and throw an error if it is not enabled for RasterFrames. + """ + sc = SparkContext._active_spark_context + if not hasattr(sc, '_rf_context'): + raise AttributeError( + "RasterFrames have not been enabled for the active session. Call 'SparkSession.withRasterFrames()'.") + return sc._rf_context + + @staticmethod + def call(name, *args): + f = RFContext.active().lookup(name) + return f(*args) + + @staticmethod + def _jvm_mirror(): + """ + Get the active Scala PyRFContext and throw an error if it is not enabled for RasterFrames. + """ + return RFContext.active()._jrfctx + diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py b/pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py new file mode 100644 index 000000000..c1fe857d4 --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py @@ -0,0 +1,135 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import pyrasterframes.rf_types + + +def tile_to_png(tile, fig_size=None): + """ Provide image of Tile.""" + if tile.cells is None: + return None + + import io + from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas + from matplotlib.figure import Figure + + # Set up matplotlib objects + nominal_size = 2 # approx full size for a 256x256 tile + if fig_size is None: + fig_size = (nominal_size, nominal_size) + + fig = Figure(figsize=fig_size) + canvas = FigureCanvas(fig) + axis = fig.add_subplot(1, 1, 1) + + data = tile.cells + + axis.imshow(data) + axis.set_aspect('equal') + axis.xaxis.set_ticks([]) + axis.yaxis.set_ticks([]) + + axis.set_title('{}, {}'.format(tile.dimensions(), tile.cell_type.__repr__()), + fontsize=fig_size[0]*4) # compact metadata as title + + with io.BytesIO() as output: + canvas.print_png(output) + return output.getvalue() + + +def tile_to_html(tile, fig_size=None): + """ Provide HTML string representation of Tile image.""" + import base64 + b64_img_html = '' + png_bits = tile_to_png(tile, fig_size) + b64_png = base64.b64encode(png_bits).decode('utf-8').replace('\n', '') + return b64_img_html.format(b64_png) + + +def pandas_df_to_html(df): + """Provide HTML formatting for pandas.DataFrame with rf_types.Tile in the columns. """ + import pandas as pd + # honor the existing options on display + if not pd.get_option("display.notebook_repr_html"): + return None + + if len(df) == 0: + return df._repr_html_() + + tile_cols = [] + for c in df.columns: + if isinstance(df.iloc[0][c], pyrasterframes.rf_types.Tile): # if the first is a Tile try formatting + tile_cols.append(c) + + def _safe_tile_to_html(t): + if isinstance(t, pyrasterframes.rf_types.Tile): + return tile_to_html(t, fig_size=(2, 2)) + else: + # handles case where objects in a column are not all Tile type + return t.__repr__() + + # dict keyed by column with custom rendering function + formatter = {c: _safe_tile_to_html for c in tile_cols} + + # This is needed to avoid our tile being rendered as `') + + def with_no_data_value(self, no_data): + if self.has_no_data() and self.no_data_value() == no_data: + return self + if self.is_floating_point(): + no_data = str(float(no_data)) + else: + no_data = str(int(no_data)) + return CellType(self.base_cell_type_name() + 'ud' + no_data) + + def __eq__(self, other): + if type(other) is type(self): + return self.cell_type_name == other.cell_type_name + else: + return False + + def __str__(self): + return "CellType({}, {})".format(self.cell_type_name, self.no_data_value()) + + def __repr__(self): + return self.cell_type_name + + +class Tile(object): + def __init__(self, cells, cell_type=None): + if cell_type is None: + # infer cell type from the cells dtype and whether or not it is masked + ct = CellType.from_numpy_dtype(cells.dtype) + if isinstance(cells, np.ma.MaskedArray): + ct = ct.with_no_data_value(cells.fill_value) + self.cell_type = ct + else: + self.cell_type = cell_type + self.cells = cells.astype(self.cell_type.to_numpy_dtype()) + + if self.cell_type.has_no_data(): + nd_value = self.cell_type.no_data_value() + if np.isnan(nd_value): + self.cells = np.ma.masked_invalid(self.cells) + else: + # if the value in the array is `nd_value`, it is masked as nodata + self.cells = np.ma.masked_equal(self.cells, nd_value) + + def __eq__(self, other): + if type(other) is type(self): + return self.cell_type == other.cell_type and \ + np.ma.allequal(self.cells, other.cells, fill_value=True) + else: + return False + + def __str__(self): + return "Tile(dimensions={}, cell_type={}, cells=\n{})" \ + .format(self.dimensions(), self.cell_type, self.cells) + + def __repr__(self): + return "Tile({}, {})" \ + .format(repr(self.cells), repr(self.cell_type)) + + def __add__(self, right): + if isinstance(right, Tile): + other = right.cells + else: + other = right + + return Tile(np.add(self.cells, other)) + + def __sub__(self, right): + if isinstance(right, Tile): + other = right.cells + else: + other = right + return Tile(np.subtract(self.cells, other)) + + def __mul__(self, right): + if isinstance(right, Tile): + other = right.cells + else: + other = right + return Tile(np.multiply(self.cells, other)) + + def __truediv__(self, right): + if isinstance(right, Tile): + other = right.cells + else: + other = right + return Tile(np.true_divide(self.cells, other)) + + def __div__(self, right): + return self.__truediv__(right) + + def __matmul__(self, right): + if isinstance(right, Tile): + other = right.cells + else: + other = right + return Tile(np.matmul(self.cells, other)) + + def dimensions(self): + """ Return a list of cols, rows as is conventional in GeoTrellis and RasterFrames.""" + return [self.cells.shape[1], self.cells.shape[0]] + + def _repr_png_(self): + """Provide default PNG rendering in IPython and Jupyter""" + from pyrasterframes.rf_ipython import tile_to_png + return tile_to_png(self) + + +class TileUDT(UserDefinedType): + @classmethod + def sqlType(cls): + """ + Mirrors `schema` in scala companion object org.apache.spark.sql.rf.TileUDT + """ + return StructType([ + StructField("cell_context", StructType([ + StructField("cellType", StructType([ + StructField("cellTypeName", StringType(), False) + ]), False), + StructField("dimensions", StructType([ + StructField("cols", ShortType(), False), + StructField("rows", ShortType(), False) + ]), False), + ]), False), + StructField("cell_data", StructType([ + StructField("cells", BinaryType(), True), + StructField("ref", StructType([ + StructField("source", RasterSourceUDT(), False), + StructField("bandIndex", IntegerType(), False), + StructField("subextent", StructType([ + StructField("xmin", DoubleType(), False), + StructField("ymin", DoubleType(), False), + StructField("xmax", DoubleType(), False), + StructField("ymax", DoubleType(), False) + ]), True) + ]), True) + ]), False) + ]) + + @classmethod + def module(cls): + return 'pyrasterframes.rf_types' + + @classmethod + def scalaUDT(cls): + return 'org.apache.spark.sql.rf.TileUDT' + + def serialize(self, tile): + cells = bytearray(tile.cells.flatten().tobytes()) + row = [ + # cell_context + [ + [tile.cell_type.cell_type_name], + tile.dimensions() + ], + # cell_data + [ + # cells + cells, + None + ] + ] + return row + + def deserialize(self, datum): + """ + Convert catalyst representation of Tile to Python version. NB: This is expensive. + :param datum: + :return: A Tile object from row data. + """ + cell_type = CellType(datum.cell_context.cellType.cellTypeName) + cols = datum.cell_context.dimensions.cols + rows = datum.cell_context.dimensions.rows + cell_data_bytes = datum.cell_data.cells + if cell_data_bytes is None: + if datum.cell_data.ref is None: + raise Exception("Invalid Tile structure. Missing cells and reference") + else: + payload = datum.cell_data.ref + cell_data_bytes = RFContext.active()._resolve_raster_ref(payload) + + if cell_data_bytes is None: + raise Exception("Unable to fetch cell data from: " + repr(datum)) + + try: + as_numpy = np.frombuffer(cell_data_bytes, dtype=cell_type.to_numpy_dtype()) + reshaped = as_numpy.reshape((rows, cols)) + t = Tile(reshaped, cell_type) + except ValueError as e: + raise ValueError({ + "cell_type": cell_type, + "cols": cols, + "rows": rows, + "cell_data.length": len(cell_data_bytes), + "cell_data.type": type(cell_data_bytes), + "cell_data.values": repr(cell_data_bytes) + }, e) + return t + + deserialize.__safe_for_unpickling__ = True + + +Tile.__UDT__ = TileUDT() + + +class TileExploder(JavaTransformer, JavaMLReadable, JavaMLWritable): + """ + Python wrapper for TileExploder.scala + """ + + def __init__(self): + super(TileExploder, self).__init__() + self._java_obj = self._new_java_obj("org.locationtech.rasterframes.ml.TileExploder", self.uid) + + +class NoDataFilter(JavaTransformer, HasInputCols, JavaMLReadable, JavaMLWritable): + """ + Python wrapper for NoDataFilter.scala + """ + + def __init__(self): + super(NoDataFilter, self).__init__() + self._java_obj = self._new_java_obj("org.locationtech.rasterframes.ml.NoDataFilter", self.uid) + diff --git a/pyrasterframes/src/main/python/pyrasterframes/utils.py b/pyrasterframes/src/main/python/pyrasterframes/utils.py new file mode 100644 index 000000000..e1ffd4074 --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/utils.py @@ -0,0 +1,100 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import glob +from pyspark.sql import SparkSession +import os +import sys +from . import RFContext + +__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version"] + + +def find_pyrasterframes_jar_dir(): + """ + Locates the directory where JVM libraries for Spark are stored. + :return: path to jar director as a string + """ + jar_dir = None + + if sys.version < "3": + import imp + try: + module_home = imp.find_module("pyrasterframes")[1] # path + jar_dir = os.path.join(module_home, 'jars') + except ImportError: + pass + + else: + from importlib.util import find_spec + try: + module_home = find_spec("pyrasterframes").origin + jar_dir = os.path.join(os.path.dirname(module_home), 'jars') + except ImportError: + pass + + # Case for when we're running from source build + if jar_dir is None or not os.path.exists(jar_dir): + def pdir(curr): + return os.path.dirname(curr) + + here = pdir(os.path.realpath(__file__)) + target_dir = pdir(pdir(here)) + # See if we're running outside of sbt build and adjust + if os.path.basename(target_dir) != "target": + target_dir = os.path.join(pdir(pdir(target_dir)), 'target') + jar_dir = os.path.join(target_dir, 'scala-2.11') + + return os.path.realpath(jar_dir) + + +def find_pyrasterframes_assembly(): + jar_dir = find_pyrasterframes_jar_dir() + jarpath = glob.glob(os.path.join(jar_dir, 'pyrasterframes-assembly*.jar')) + + if not len(jarpath) == 1: + raise RuntimeError(""" +Expected to find exactly one assembly. Found '{}' instead. +Try running 'sbt pyrasterframes/clean pyrasterframes/package' first. """.format(jarpath)) + return jarpath[0] + + +def create_rf_spark_session(master="local[*]"): + """ Create a SparkSession with pyrasterframes enabled and configured. """ + jar_path = find_pyrasterframes_assembly() + + spark = (SparkSession.builder + .master(master) + .appName("RasterFrames") + .config('spark.jars', jar_path) + .withKryoSerialization() + .getOrCreate()) + + try: + spark.withRasterFrames() + return spark + except TypeError as te: + print("Error setting up SparkSession; cannot find the pyrasterframes assembly jar\n", te) + return None + + +def gdal_version(): + fcn = RFContext.active().lookup("buildInfo") + return fcn()["GDAL"] diff --git a/pyrasterframes/src/main/python/pyrasterframes/version.py b/pyrasterframes/src/main/python/pyrasterframes/version.py new file mode 100644 index 000000000..273686206 --- /dev/null +++ b/pyrasterframes/src/main/python/pyrasterframes/version.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Translating Java version from version.sbt to PEP440 norms +__version__ = '0.8.0' diff --git a/pyrasterframes/src/main/python/requirements.txt b/pyrasterframes/src/main/python/requirements.txt new file mode 100644 index 000000000..536636f33 --- /dev/null +++ b/pyrasterframes/src/main/python/requirements.txt @@ -0,0 +1,8 @@ +pytz +Shapely>=1.6.0 +pyspark==2.3.3 # See issue # 154 +numpy>=1.7 +pandas>=0.25.0 +matplotlib<3.0.0 # no python 2.7 support after v2.x.x +ipython==6.2.1 +rasterio>=1.0.0 \ No newline at end of file diff --git a/pyrasterframes/src/main/python/setup.cfg b/pyrasterframes/src/main/python/setup.cfg new file mode 100644 index 000000000..8088676c6 --- /dev/null +++ b/pyrasterframes/src/main/python/setup.cfg @@ -0,0 +1,13 @@ +[metadata] +license_files = LICENSE.txt + +[bdist_wheel] +universal=1 + +[aliases] +test=pytest + +[tool:pytest] +addopts = --verbose +testpaths = tests +python_files = *.py diff --git a/pyrasterframes/src/main/python/setup.py b/pyrasterframes/src/main/python/setup.py new file mode 100644 index 000000000..2e5b7d4ed --- /dev/null +++ b/pyrasterframes/src/main/python/setup.py @@ -0,0 +1,210 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Always prefer setuptools over distutils +from setuptools import setup +from os import path +import sys +from glob import glob +from io import open +import distutils.cmd + +try: + exec(open('pyrasterframes/version.py').read()) # executable python script contains __version__; credit pyspark +except IOError: + print("Run setup via `sbt 'pySetup arg1 arg2'` to ensure correct access to all source files and binaries.") + sys.exit(-1) + + +VERSION = __version__ + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + readme = f.read() + + +def _divided(msg): + divider = ('-' * 50) + return divider + '\n' + msg + '\n' + divider + + +class PweaveDocs(distutils.cmd.Command): + """A custom command to run documentation scripts through pweave.""" + description = 'Pweave PyRasterFrames documentation scripts' + user_options = [ + # The format is (long option, short option, description). + ('files=', 's', 'Specific files to pweave. Defaults to all in `docs` directory.'), + ('format=', 'f', 'Output format type. Defaults to `markdown`'), + ('quick=', 'q', 'Check to see if the source file is newer than existing output before building. Defaults to `False`.') + ] + + + def initialize_options(self): + """Set default values for options.""" + # Each user option must be listed here with their default value. + self.files = filter( + lambda x: not path.basename(x)[:1] == '_', + glob(path.join(here, 'docs', '*.pymd')) + ) + self.format = 'markdown' + self.quick = False + + def finalize_options(self): + """Post-process options.""" + import re + if isinstance(self.files, str): + self.files = filter(lambda s: len(s) > 0, re.split(',', self.files)) + # `html` doesn't do quite what one expects... only replaces code blocks, leaving markdown in place + print("format.....", self.format) + if self.format == 'html': + self.format = 'pandoc2html' + if isinstance(self.quick, str): + self.quick = self.quick == 'True' or self.quick == 'true' + + def dest_file(self, src_file): + return path.splitext(src_file)[0] + '.md' + + def run(self): + """Run pweave.""" + import traceback + import pweave + from docs import PegdownMarkdownFormatter + + bad_words = ["Error"] + pweave.rcParams["chunk"]["defaultoptions"].update({'wrap': False, 'dpi': 175}) + if self.format == 'markdown': + pweave.PwebFormats.formats['markdown'] = { + 'class': PegdownMarkdownFormatter, + 'description': 'Pegdown compatible markdown' + } + + for file in sorted(self.files, reverse=True): + name = path.splitext(path.basename(file))[0] + dest = self.dest_file(file) + + if (not self.quick) or (not path.exists(dest)) or (path.getmtime(dest) < path.getmtime(file)): + print(_divided('Running %s' % name)) + try: + pweave.weave(file=str(file), doctype=self.format, cache=True) + if self.format == 'markdown': + if not path.exists(dest): + raise FileNotFoundError("Markdown file '%s' didn't get created as expected" % dest) + with open(dest, "r") as result: + for (n, line) in enumerate(result): + for word in bad_words: + if word in line: + raise ChildProcessError("Error detected on line %s in %s:\n%s" % (n + 1, dest, line)) + + except Exception: + print(_divided('%s Failed:' % file)) + print(traceback.format_exc()) + exit(1) + else: + print(_divided('Skipping %s' % name)) + + +class PweaveNotebooks(PweaveDocs): + def initialize_options(self): + super().initialize_options() + self.format = 'notebook' + + def dest_file(self, src_file): + return path.splitext(src_file)[0] + '.ipynb' + +setup( + name='pyrasterframes', + description='Access and process geospatial raster data in PySpark DataFrames', + long_description=readme, + long_description_content_type='text/markdown', + version=VERSION, + author='Astraea, Inc.', + author_email='info@astraea.earth', + license='Apache 2', + url='https://rasterframes.io', + project_urls={ + 'Bug Reports': 'https://github.com/locationtech/rasterframes/issues', + 'Source': 'https://github.com/locationtech/rasterframes', + }, + install_requires=[ + 'pytz', + 'Shapely>=1.6.0', + 'pyspark<2.4', + 'numpy>=1.7', + 'pandas>=0.25.0', + ], + setup_requires=[ + 'pytz', + 'Shapely>=1.6.0', + 'pyspark<2.4', + 'numpy>=1.7', + 'matplotlib<3.0.0', + 'pandas>=0.25.0', + 'geopandas', + 'requests', + 'pytest-runner', + 'setuptools>=0.8', + 'ipython==6.2.1', + 'ipykernel==4.8.0', + 'Pweave==0.30.3', + 'fiona==1.8.6', + 'rasterio>=1.0.0', # for docs + ], + tests_require=[ + 'pytest==3.4.2', + 'pypandoc', + 'numpy>=1.7', + 'Shapely>=1.6.0', + 'pandas>=0.25.0', + 'rasterio>=1.0.0', + 'boto3', + 'Pweave==0.30.3', + ], + packages=[ + 'pyrasterframes', + 'geomesa_pyspark', + 'pyrasterframes.jars', + ], + package_dir={ + 'pyrasterframes.jars': 'deps/jars' + }, + package_data={ + 'pyrasterframes.jars': ['*.jar'] + }, + include_package_data=True, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Other Environment', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: Unix', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries', + 'Topic :: Scientific/Engineering :: GIS', + 'Topic :: Multimedia :: Graphics :: Graphics Conversion', + ], + zip_safe=False, + test_suite="pytest-runner", + cmdclass={ + 'pweave': PweaveDocs, + 'notebooks': PweaveNotebooks + } +) diff --git a/pyrasterframes/src/main/python/tests/ExploderTests.py b/pyrasterframes/src/main/python/tests/ExploderTests.py new file mode 100644 index 000000000..05ff958ae --- /dev/null +++ b/pyrasterframes/src/main/python/tests/ExploderTests.py @@ -0,0 +1,59 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from . import TestEnvironment + +from pyrasterframes.rasterfunctions import * +from pyrasterframes.rf_types import * +from pyrasterframes import TileExploder + +from pyspark.ml.feature import VectorAssembler +from pyspark.ml import Pipeline +from pyspark.sql.functions import * + +import unittest + + +class ExploderTests(TestEnvironment): + + @unittest.skip("See issue https://github.com/locationtech/rasterframes/issues/163") + def test_tile_exploder_pipeline_for_prt(self): + # NB the tile is a Projected Raster Tile + df = self.spark.read.raster(self.img_uri) + t_col = 'proj_raster' + self.assertTrue(t_col in df.columns) + + assembler = VectorAssembler().setInputCols([t_col]) + pipe = Pipeline().setStages([TileExploder(), assembler]) + pipe_model = pipe.fit(df) + tranformed_df = pipe_model.transform(df) + self.assertTrue(tranformed_df.count() > df.count()) + + def test_tile_exploder_pipeline_for_tile(self): + t_col = 'tile' + df = self.spark.read.raster(self.img_uri) \ + .withColumn(t_col, rf_tile('proj_raster')) \ + .drop('proj_raster') + + assembler = VectorAssembler().setInputCols([t_col]) + pipe = Pipeline().setStages([TileExploder(), assembler]) + pipe_model = pipe.fit(df) + tranformed_df = pipe_model.transform(df) + self.assertTrue(tranformed_df.count() > df.count()) diff --git a/pyrasterframes/src/main/python/tests/GeoTiffWriterTests.py b/pyrasterframes/src/main/python/tests/GeoTiffWriterTests.py new file mode 100644 index 000000000..8fa30a7b2 --- /dev/null +++ b/pyrasterframes/src/main/python/tests/GeoTiffWriterTests.py @@ -0,0 +1,89 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import tempfile + +from . import TestEnvironment +import rasterio + + +class GeoTiffWriter(TestEnvironment): + + @staticmethod + def _tmpfile(): + return os.path.join(tempfile.gettempdir(), "pyrf-test.tif") + + def test_identity_write(self): + rf = self.spark.read.geotiff(self.img_uri) + rf_count = rf.count() + self.assertTrue(rf_count > 0) + + dest = self._tmpfile() + rf.write.geotiff(dest) + + rf2 = self.spark.read.geotiff(dest) + + self.assertEqual(rf2.count(), rf.count()) + + os.remove(dest) + + def test_unstructured_write(self): + rf = self.spark.read.raster(self.img_uri) + dest_file = self._tmpfile() + rf.write.geotiff(dest_file, crs='EPSG:32616') + + rf2 = self.spark.read.raster(dest_file) + self.assertEqual(rf2.count(), rf.count()) + + with rasterio.open(self.img_uri) as source: + with rasterio.open(dest_file) as dest: + self.assertEqual((dest.width, dest.height), (source.width, source.height)) + self.assertEqual(dest.bounds, source.bounds) + self.assertEqual(dest.crs, source.crs) + + os.remove(dest_file) + + def test_unstructured_write_schemeless(self): + # should be able to write a projected raster tile column to path like '/data/foo/file.tif' + from pyrasterframes.rasterfunctions import rf_agg_stats, rf_crs + rf = self.spark.read.raster(self.img_uri) + max = rf.agg(rf_agg_stats('proj_raster').max.alias('max')).first()['max'] + crs = rf.select(rf_crs('proj_raster').crsProj4.alias('c')).first()['c'] + + dest_file = self._tmpfile() + self.assertTrue(not dest_file.startswith('file://')) + rf.write.geotiff(dest_file, crs=crs) + + with rasterio.open(dest_file) as src: + self.assertEqual(src.read().max(), max) + + os.remove(dest_file) + + def test_downsampled_write(self): + rf = self.spark.read.raster(self.img_uri) + dest = self._tmpfile() + rf.write.geotiff(dest, crs='EPSG:32616', raster_dimensions=(128, 128)) + + with rasterio.open(dest) as f: + self.assertEqual((f.width, f.height), (128, 128)) + + os.remove(dest) + diff --git a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py new file mode 100644 index 000000000..f682e2609 --- /dev/null +++ b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py @@ -0,0 +1,570 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import unittest + +import numpy as np +from pyrasterframes.rasterfunctions import * +from pyrasterframes.rf_types import * +from pyspark.sql import SQLContext +from pyspark.sql.functions import * +from . import TestEnvironment + + +class CellTypeHandling(unittest.TestCase): + + def test_is_raw(self): + self.assertTrue(CellType("float32raw").is_raw()) + self.assertFalse(CellType("float64ud1234").is_raw()) + self.assertFalse(CellType("float32").is_raw()) + self.assertTrue(CellType("int8raw").is_raw()) + self.assertFalse(CellType("uint16d12").is_raw()) + self.assertFalse(CellType("int32").is_raw()) + + def test_is_floating_point(self): + self.assertTrue(CellType("float32raw").is_floating_point()) + self.assertTrue(CellType("float64ud1234").is_floating_point()) + self.assertTrue(CellType("float32").is_floating_point()) + self.assertFalse(CellType("int8raw").is_floating_point()) + self.assertFalse(CellType("uint16d12").is_floating_point()) + self.assertFalse(CellType("int32").is_floating_point()) + + def test_cell_type_no_data(self): + import math + self.assertIsNone(CellType.bool().no_data_value()) + + self.assertTrue(CellType.int8().has_no_data()) + self.assertEqual(CellType.int8().no_data_value(), -128) + + self.assertTrue(CellType.uint8().has_no_data()) + self.assertEqual(CellType.uint8().no_data_value(), 0) + + self.assertTrue(CellType.int16().has_no_data()) + self.assertEqual(CellType.int16().no_data_value(), -32768) + + self.assertTrue(CellType.uint16().has_no_data()) + self.assertEqual(CellType.uint16().no_data_value(), 0) + + self.assertTrue(CellType.float32().has_no_data()) + self.assertTrue(np.isnan(CellType.float32().no_data_value())) + + self.assertEqual(CellType("float32ud-98").no_data_value(), -98.0) + self.assertEqual(CellType("float32ud-98").no_data_value(), -98) + self.assertEqual(CellType("int32ud-98").no_data_value(), -98.0) + self.assertEqual(CellType("int32ud-98").no_data_value(), -98) + + self.assertTrue(math.isnan(CellType.float64().no_data_value())) + self.assertEqual(CellType.uint8().no_data_value(), 0) + + def test_cell_type_conversion(self): + for ct in rf_cell_types(): + self.assertEqual(ct.to_numpy_dtype(), + CellType.from_numpy_dtype(ct.to_numpy_dtype()).to_numpy_dtype(), + "dtype comparison for " + str(ct)) + if not ct.is_raw(): + self.assertEqual(ct, + CellType.from_numpy_dtype(ct.to_numpy_dtype()), + "GTCellType comparison for " + str(ct)) + else: + ct_ud = ct.with_no_data_value(99) + self.assertEqual(ct_ud.base_cell_type_name(), + repr(CellType.from_numpy_dtype(ct_ud.to_numpy_dtype())), + "GTCellType comparison for " + str(ct_ud) + ) + + +class UDT(TestEnvironment): + + def setUp(self): + self.create_layer() + + def test_mask_no_data(self): + t1 = Tile(np.array([[1, 2], [3, 4]]), CellType("int8ud3")) + self.assertTrue(t1.cells.mask[1][0]) + self.assertIsNotNone(t1.cells[1][1]) + self.assertEqual(len(t1.cells.compressed()), 3) + + t2 = Tile(np.array([[1.0, 2.0], [float('nan'), 4.0]]), CellType.float32()) + self.assertEqual(len(t2.cells.compressed()), 3) + self.assertTrue(t2.cells.mask[1][0]) + self.assertIsNotNone(t2.cells[1][1]) + + def test_tile_udt_serialization(self): + from pyspark.sql.types import StructType, StructField + + udt = TileUDT() + cell_types = (ct for ct in rf_cell_types() if not (ct.is_raw() or ("bool" in ct.base_cell_type_name()))) + + for ct in cell_types: + cells = (100 + np.random.randn(3, 3) * 100).astype(ct.to_numpy_dtype()) + + if ct.is_floating_point(): + nd = 33.0 + else: + nd = 33 + + cells[1][1] = nd + a_tile = Tile(cells, ct.with_no_data_value(nd)) + round_trip = udt.fromInternal(udt.toInternal(a_tile)) + self.assertEquals(a_tile, round_trip, "round-trip serialization for " + str(ct)) + + schema = StructType([StructField("tile", TileUDT(), False)]) + df = self.spark.createDataFrame([{"tile": a_tile}], schema) + + long_trip = df.first()["tile"] + self.assertEqual(long_trip, a_tile) + + def test_udf_on_tile_type_input(self): + import numpy.testing + df = self.spark.read.raster(self.img_uri) + rf = self.rf + + # create trivial UDF that does something we already do with raster_Functions + @udf('integer') + def my_udf(t): + a = t.cells + return a.size # same as rf_dimensions.cols * rf_dimensions.rows + + rf_result = rf.select( + (rf_dimensions('tile').cols.cast('int') * rf_dimensions('tile').rows.cast('int')).alias('expected'), + my_udf('tile').alias('result')).toPandas() + + numpy.testing.assert_array_equal( + rf_result.expected.tolist(), + rf_result.result.tolist() + ) + + df_result = df.select( + (rf_dimensions(df.proj_raster).cols.cast('int') * rf_dimensions(df.proj_raster).rows.cast('int') - + my_udf(rf_tile(df.proj_raster))).alias('result') + ).toPandas() + + numpy.testing.assert_array_equal( + np.zeros(len(df_result)), + df_result.result.tolist() + ) + + def test_udf_on_tile_type_output(self): + import numpy.testing + + rf = self.rf + + # create a trivial UDF that does something we already do with a raster_functions + @udf(TileUDT()) + def my_udf(t): + import numpy as np + return Tile(np.log1p(t.cells)) + + rf_result = rf.select( + rf_tile_max( + rf_local_subtract( + my_udf(rf.tile), + rf_log1p(rf.tile) + ) + ).alias('expect_zeros') + ).collect() + + # almost equal because of different implemenations under the hoods: C (numpy) versus Java (rf_) + numpy.testing.assert_almost_equal( + [r['expect_zeros'] for r in rf_result], + [0.0 for _ in rf_result], + decimal=6 + ) + + def test_no_data_udf_handling(self): + from pyspark.sql.types import StructType, StructField + + t1 = Tile(np.array([[1, 2], [0, 4]]), CellType.uint8()) + self.assertEqual(t1.cell_type.to_numpy_dtype(), np.dtype("uint8")) + e1 = Tile(np.array([[2, 3], [0, 5]]), CellType.uint8()) + schema = StructType([StructField("tile", TileUDT(), False)]) + df = self.spark.createDataFrame([{"tile": t1}], schema) + + @udf(TileUDT()) + def increment(t): + return t + 1 + + r1 = df.select(increment(df.tile).alias("inc")).first()["inc"] + self.assertEqual(r1, e1) + + def test_udf_np_implicit_type_conversion(self): + import math + import pandas + + a1 = np.array([[1, 2], [0, 4]]) + t1 = Tile(a1, CellType.uint8()) + exp_array = a1.astype('>f8') + + @udf(TileUDT()) + def times_pi(t): + return t * math.pi + + @udf(TileUDT()) + def divide_pi(t): + return t / math.pi + + @udf(TileUDT()) + def plus_pi(t): + return t + math.pi + + @udf(TileUDT()) + def less_pi(t): + return t - math.pi + + df = self.spark.createDataFrame(pandas.DataFrame([{"tile": t1}])) + r1 = df.select( + less_pi(divide_pi(times_pi(plus_pi(df.tile)))) + ).first()[0] + + self.assertTrue(np.all(r1.cells == exp_array)) + self.assertEqual(r1.cells.dtype, exp_array.dtype) + + +class TileOps(TestEnvironment): + + def setUp(self): + from pyspark.sql import Row + # convenience so we can assert around Tile() == Tile() + self.t1 = Tile(np.array([[1, 2], + [3, 4]]), CellType.int8().with_no_data_value(3)) + self.t2 = Tile(np.array([[1, 2], + [3, 4]]), CellType.int8().with_no_data_value(1)) + self.t3 = Tile(np.array([[1, 2], + [-3, 4]]), CellType.int8().with_no_data_value(3)) + + self.df = self.spark.createDataFrame([Row(t1=self.t1, t2=self.t2, t3=self.t3)]) + + def test_addition(self): + e1 = np.ma.masked_equal(np.array([[5, 6], + [7, 8]]), 7) + self.assertTrue(np.array_equal((self.t1 + 4).cells, e1)) + + e2 = np.ma.masked_equal(np.array([[3, 4], + [3, 8]]), 3) + r2 = (self.t1 + self.t2).cells + self.assertTrue(np.ma.allequal(r2, e2)) + + col_result = self.df.select(rf_local_add('t1', 't3').alias('sum')).first() + self.assertEqual(col_result.sum, self.t1 + self.t3) + + def test_multiplication(self): + e1 = np.ma.masked_equal(np.array([[4, 8], + [12, 16]]), 12) + + self.assertTrue(np.array_equal((self.t1 * 4).cells, e1)) + + e2 = np.ma.masked_equal(np.array([[3, 4], [3, 16]]), 3) + r2 = (self.t1 * self.t2).cells + self.assertTrue(np.ma.allequal(r2, e2)) + + r3 = self.df.select(rf_local_multiply('t1', 't3').alias('r3')).first().r3 + self.assertEqual(r3, self.t1 * self.t3) + + def test_subtraction(self): + t3 = self.t1 * 4 + r1 = t3 - self.t1 + # note careful construction of mask value and dtype above + e1 = Tile(np.ma.masked_equal(np.array([[4 - 1, 8 - 2], + [3, 16 - 4]], dtype='int8'), + 3, ) + ) + self.assertTrue(r1 == e1, + "{} does not equal {}".format(r1, e1)) + # put another way + self.assertTrue(r1 == self.t1 * 3, + "{} does not equal {}".format(r1, self.t1 * 3)) + + def test_division(self): + t3 = self.t1 * 9 + r1 = t3 / 9 + self.assertTrue(np.array_equal(r1.cells, self.t1.cells), + "{} does not equal {}".format(r1, self.t1)) + + r2 = (self.t1 / self.t1).cells + self.assertTrue(np.array_equal(r2, np.array([[1,1], [1, 1]], dtype=r2.dtype))) + + def test_matmul(self): + # if sys.version >= '3.5': # per https://docs.python.org/3.7/library/operator.html#operator.matmul new in 3.5 + # r1 = self.t1 @ self.t2 + r1 = self.t1.__matmul__(self.t2) + + # The behavior of np.matmul with masked arrays is not well documented + # it seems to treat the 2nd arg as if not a MaskedArray + e1 = Tile(np.matmul(self.t1.cells, self.t2.cells), r1.cell_type) + + self.assertTrue(r1 == e1, "{} was not equal to {}".format(r1, e1)) + self.assertEqual(r1, e1) + + +class PandasInterop(TestEnvironment): + + def setUp(self): + self.create_layer() + + def test_pandas_conversion(self): + import pandas as pd + # pd.options.display.max_colwidth = 256 + cell_types = (ct for ct in rf_cell_types() if not (ct.is_raw() or ("bool" in ct.base_cell_type_name()))) + tiles = [Tile(np.random.randn(5, 5) * 100, ct) for ct in cell_types] + in_pandas = pd.DataFrame({ + 'tile': tiles + }) + + in_spark = self.spark.createDataFrame(in_pandas) + out_pandas = in_spark.select(rf_identity('tile').alias('tile')).toPandas() + self.assertTrue(out_pandas.equals(in_pandas), str(in_pandas) + "\n\n" + str(out_pandas)) + + def test_extended_pandas_ops(self): + import pandas as pd + + self.assertIsInstance(self.rf.sql_ctx, SQLContext) + + # Try to collect self.rf which is read from a geotiff + rf_collect = self.rf.take(2) + self.assertTrue( + all([isinstance(row.tile.cells, np.ndarray) for row in rf_collect])) + + # Try to create a tile from numpy. + self.assertEqual(Tile(np.random.randn(10, 10), CellType.int8()).dimensions(), [10, 10]) + + tiles = [Tile(np.random.randn(10, 12), CellType.float64()) for _ in range(3)] + to_spark = pd.DataFrame({ + 't': tiles, + 'b': ['a', 'b', 'c'], + 'c': [1, 2, 4], + }) + rf_maybe = self.spark.createDataFrame(to_spark) + + # rf_maybe.select(rf_render_matrix(rf_maybe.t)).show(truncate=False) + + # Try to do something with it. + sums = to_spark.t.apply(lambda a: a.cells.sum()).tolist() + maybe_sums = rf_maybe.select(rf_tile_sum(rf_maybe.t).alias('tsum')) + maybe_sums = [r.tsum for r in maybe_sums.collect()] + np.testing.assert_almost_equal(maybe_sums, sums, 12) + + # Test round trip for an array + simple_array = Tile(np.array([[1, 2], [3, 4]]), CellType.float64()) + to_spark_2 = pd.DataFrame({ + 't': [simple_array] + }) + + rf_maybe_2 = self.spark.createDataFrame(to_spark_2) + #print("RasterFrameLayer `show`:") + #rf_maybe_2.select(rf_render_matrix(rf_maybe_2.t).alias('t')).show(truncate=False) + + pd_2 = rf_maybe_2.toPandas() + array_back_2 = pd_2.iloc[0].t + #print("Array collected from toPandas output\n", array_back_2) + + self.assertIsInstance(array_back_2, Tile) + np.testing.assert_equal(array_back_2.cells, simple_array.cells) + + +class RasterJoin(TestEnvironment): + + def setUp(self): + self.create_layer() + + def test_raster_join(self): + # re-read the same source + rf_prime = self.spark.read.geotiff(self.img_uri) \ + .withColumnRenamed('tile', 'tile2').alias('rf_prime') + + rf_joined = self.rf.raster_join(rf_prime) + + self.assertTrue(rf_joined.count(), self.rf.count()) + self.assertTrue(len(rf_joined.columns) == len(self.rf.columns) + len(rf_prime.columns) - 2) + + rf_joined_2 = self.rf.raster_join(rf_prime, self.rf.extent, self.rf.crs, rf_prime.extent, rf_prime.crs) + self.assertTrue(rf_joined_2.count(), self.rf.count()) + self.assertTrue(len(rf_joined_2.columns) == len(self.rf.columns) + len(rf_prime.columns) - 2) + + # this will bring arbitrary additional data into join; garbage result + join_expression = self.rf.extent.xmin == rf_prime.extent.xmin + rf_joined_3 = self.rf.raster_join(rf_prime, self.rf.extent, self.rf.crs, + rf_prime.extent, rf_prime.crs, + join_expression) + self.assertTrue(rf_joined_3.count(), self.rf.count()) + self.assertTrue(len(rf_joined_3.columns) == len(self.rf.columns) + len(rf_prime.columns) - 2) + + # throws if you don't pass in all expected columns + with self.assertRaises(AssertionError): + self.rf.raster_join(rf_prime, join_exprs=self.rf.extent) + + +class RasterSource(TestEnvironment): + + def test_handle_lazy_eval(self): + df = self.spark.read.raster(self.img_uri) + ltdf = df.select('proj_raster') + self.assertGreater(ltdf.count(), 0) + self.assertIsNotNone(ltdf.first()) + + tdf = df.select(rf_tile('proj_raster')) + self.assertGreater(tdf.count(), 0) + self.assertIsNotNone(tdf.first()) + + def test_strict_eval(self): + df_lazy = self.spark.read.raster(self.img_uri, lazy_tiles=True) + # when doing Show on a lazy tile we will see something like RasterRefTile(RasterRef(JVMGeoTiffRasterSource(... + # use this trick to get the `show` string + show_str_lazy = df_lazy.select('proj_raster')._jdf.showString(1, -1, False) + self.assertTrue('RasterRef' in show_str_lazy) + + # again for strict + df_strict = self.spark.read.raster(self.img_uri, lazy_tiles=False) + show_str_strict = df_strict.select('proj_raster')._jdf.showString(1, -1, False) + self.assertTrue('RasterRef' not in show_str_strict) + + + def test_prt_functions(self): + df = self.spark.read.raster(self.img_uri) \ + .withColumn('crs', rf_crs('proj_raster')) \ + .withColumn('ext', rf_extent('proj_raster')) \ + .withColumn('geom', rf_geometry('proj_raster')) + df.select('crs', 'ext', 'geom').first() + + def test_raster_source_reader(self): + # much the same as RasterSourceDataSourceSpec here; but using https PDS. Takes about 30s to run + + def l8path(b): + assert b in range(1, 12) + base = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/199/026/LC08_L1TP_199026_20180919_20180928_01_T1/LC08_L1TP_199026_20180919_20180928_01_T1_B{}.TIF" + return base.format(b) + + path_param = '\n'.join([l8path(b) for b in [1, 2, 3]]) # "http://foo.com/file1.tif,http://foo.com/file2.tif" + tile_size = 512 + + df = self.spark.read.raster( + tile_dimensions=(tile_size, tile_size), + paths=path_param, + lazy_tiles=True, + ).cache() + + # schema is tile_path and tile + # df.printSchema() + self.assertTrue(len(df.columns) == 2 and 'proj_raster_path' in df.columns and 'proj_raster' in df.columns) + + # the most common tile dimensions should be as passed to `options`, showing that options are correctly applied + tile_size_df = df.select(rf_dimensions(df.proj_raster).rows.alias('r'), rf_dimensions(df.proj_raster).cols.alias('c')) \ + .groupby(['r', 'c']).count().toPandas() + most_common_size = tile_size_df.loc[tile_size_df['count'].idxmax()] + self.assertTrue(most_common_size.r == tile_size and most_common_size.c == tile_size) + + # all rows are from a single source URI + path_count = df.groupby(df.proj_raster_path).count() + print(path_count.toPandas()) + self.assertTrue(path_count.count() == 3) + + def test_raster_source_reader_schemeless(self): + import os.path + path = os.path.join(self.resource_dir, "L8-B8-Robinson-IL.tiff") + self.assertTrue(not path.startswith('file://')) + df = self.spark.read.raster(path) + self.assertTrue(df.count() > 0) + + def test_raster_source_catalog_reader(self): + import pandas as pd + + scene_dict = { + 1: 'http://landsat-pds.s3.amazonaws.com/c1/L8/015/041/LC08_L1TP_015041_20190305_20190309_01_T1/LC08_L1TP_015041_20190305_20190309_01_T1_B{}.TIF', + 2: 'http://landsat-pds.s3.amazonaws.com/c1/L8/015/042/LC08_L1TP_015042_20190305_20190309_01_T1/LC08_L1TP_015042_20190305_20190309_01_T1_B{}.TIF', + 3: 'http://landsat-pds.s3.amazonaws.com/c1/L8/016/041/LC08_L1TP_016041_20190224_20190309_01_T1/LC08_L1TP_016041_20190224_20190309_01_T1_B{}.TIF', + } + + def path(scene, band): + assert band in range(1, 12) + p = scene_dict[scene] + return p.format(band) + + # Create a pandas dataframe (makes it easy to create spark df) + path_pandas = pd.DataFrame([ + {'b1': path(1, 1), 'b2': path(1, 2), 'b3': path(1, 3)}, + {'b1': path(2, 1), 'b2': path(2, 2), 'b3': path(2, 3)}, + {'b1': path(3, 1), 'b2': path(3, 2), 'b3': path(3, 3)}, + ]) + # comma separated list of column names containing URI's to read. + catalog_columns = ','.join(path_pandas.columns.tolist()) # 'b1,b2,b3' + path_table = self.spark.createDataFrame(path_pandas) + + path_df = self.spark.read.raster( + tile_dimensions=(512, 512), + catalog=path_table, + catalog_col_names=catalog_columns, + lazy_tiles=True # We'll get an OOM error if we try to read 9 scenes all at once! + ) + + self.assertTrue(len(path_df.columns) == 6) # three bands times {path, tile} + self.assertTrue(path_df.select('b1_path').distinct().count() == 3) # as per scene_dict + b1_paths_maybe = path_df.select('b1_path').distinct().collect() + b1_paths = [s.format('1') for s in scene_dict.values()] + self.assertTrue(all([row.b1_path in b1_paths for row in b1_paths_maybe])) + + def test_raster_source_catalog_reader_with_pandas(self): + import pandas as pd + import geopandas + from shapely.geometry import Point + + scene_dict = { + 1: 'http://landsat-pds.s3.amazonaws.com/c1/L8/015/041/LC08_L1TP_015041_20190305_20190309_01_T1/LC08_L1TP_015041_20190305_20190309_01_T1_B{}.TIF', + 2: 'http://landsat-pds.s3.amazonaws.com/c1/L8/015/042/LC08_L1TP_015042_20190305_20190309_01_T1/LC08_L1TP_015042_20190305_20190309_01_T1_B{}.TIF', + 3: 'http://landsat-pds.s3.amazonaws.com/c1/L8/016/041/LC08_L1TP_016041_20190224_20190309_01_T1/LC08_L1TP_016041_20190224_20190309_01_T1_B{}.TIF', + } + + def path(scene, band): + assert band in range(1, 12) + p = scene_dict[scene] + return p.format(band) + + # Create a pandas dataframe (makes it easy to create spark df) + path_pandas = pd.DataFrame([ + {'b1': path(1, 1), 'b2': path(1, 2), 'b3': path(1, 3), 'geo': Point(1, 1)}, + {'b1': path(2, 1), 'b2': path(2, 2), 'b3': path(2, 3), 'geo': Point(2, 2)}, + {'b1': path(3, 1), 'b2': path(3, 2), 'b3': path(3, 3), 'geo': Point(3, 3)}, + ]) + + # here a subtle difference with the test_raster_source_catalog_reader test, feed the DataFrame not a CSV and not an already created spark DF. + df = self.spark.read.raster( + catalog=path_pandas, + catalog_col_names=['b1', 'b2', 'b3'] + ) + self.assertEqual(len(df.columns), 7) # three path cols, three tile cols, and geo + self.assertTrue('geo' in df.columns) + self.assertTrue(df.select('b1_path').distinct().count() == 3) + + + # Same test with geopandas + geo_df = geopandas.GeoDataFrame(path_pandas, crs={'init': 'EPSG:4326'}, geometry='geo') + df2 = self.spark.read.raster( + catalog=geo_df, + catalog_col_names=['b1', 'b2', 'b3'] + ) + self.assertEqual(len(df2.columns), 7) # three path cols, three tile cols, and geo + self.assertTrue('geo' in df2.columns) + self.assertTrue(df2.select('b1_path').distinct().count() == 3) + + +def suite(): + function_tests = unittest.TestSuite() + return function_tests + + +unittest.TextTestRunner().run(suite()) diff --git a/pyrasterframes/src/main/python/tests/RasterFunctionsTests.py b/pyrasterframes/src/main/python/tests/RasterFunctionsTests.py new file mode 100644 index 000000000..fb0e0870d --- /dev/null +++ b/pyrasterframes/src/main/python/tests/RasterFunctionsTests.py @@ -0,0 +1,267 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from pyrasterframes.rasterfunctions import * +from pyrasterframes.utils import gdal_version +from pyspark.sql.functions import * + +from . import TestEnvironment + + +class RasterFunctions(TestEnvironment): + + def setUp(self): + self.create_layer() + + def test_setup(self): + self.assertEqual(self.spark.sparkContext.getConf().get("spark.serializer"), + "org.apache.spark.serializer.KryoSerializer") + print("GDAL version", gdal_version()) + + def test_identify_columns(self): + cols = self.rf.tile_columns() + self.assertEqual(len(cols), 1, '`tileColumns` did not find the proper number of columns.') + print("Tile columns: ", cols) + col = self.rf.spatial_key_column() + self.assertIsInstance(col, Column, '`spatialKeyColumn` was not found') + print("Spatial key column: ", col) + col = self.rf.temporal_key_column() + self.assertIsNone(col, '`temporalKeyColumn` should be `None`') + print("Temporal key column: ", col) + + def test_tile_creation(self): + from pyrasterframes.rf_types import CellType + + base = self.spark.createDataFrame([1, 2, 3, 4], 'integer') + tiles = base.select(rf_make_constant_tile(3, 3, 3, "int32"), rf_make_zeros_tile(3, 3, "int32"), + rf_make_ones_tile(3, 3, CellType.int32())) + tiles.show() + self.assertEqual(tiles.count(), 4) + + def test_multi_column_operations(self): + df1 = self.rf.withColumnRenamed('tile', 't1').as_layer() + df2 = self.rf.withColumnRenamed('tile', 't2').as_layer() + df3 = df1.spatial_join(df2).as_layer() + df3 = df3.withColumn('norm_diff', rf_normalized_difference('t1', 't2')) + # df3.printSchema() + + aggs = df3.agg( + rf_agg_mean('norm_diff'), + ) + aggs.show() + row = aggs.first() + + self.assertTrue(self.rounded_compare(row['rf_agg_mean(norm_diff)'], 0)) + + def test_general(self): + meta = self.rf.tile_layer_metadata() + self.assertIsNotNone(meta['bounds']) + df = self.rf.withColumn('dims', rf_dimensions('tile')) \ + .withColumn('type', rf_cell_type('tile')) \ + .withColumn('dCells', rf_data_cells('tile')) \ + .withColumn('ndCells', rf_no_data_cells('tile')) \ + .withColumn('min', rf_tile_min('tile')) \ + .withColumn('max', rf_tile_max('tile')) \ + .withColumn('mean', rf_tile_mean('tile')) \ + .withColumn('sum', rf_tile_sum('tile')) \ + .withColumn('stats', rf_tile_stats('tile')) \ + .withColumn('extent', st_extent('geometry')) \ + .withColumn('extent_geom1', st_geometry('extent')) \ + .withColumn('ascii', rf_render_ascii('tile')) \ + .withColumn('log', rf_log('tile')) \ + .withColumn('exp', rf_exp('tile')) \ + .withColumn('expm1', rf_expm1('tile')) \ + .withColumn('round', rf_round('tile')) \ + .withColumn('abs', rf_abs('tile')) + + df.first() + + def test_agg_mean(self): + mean = self.rf.agg(rf_agg_mean('tile')).first()['rf_agg_mean(tile)'] + self.assertTrue(self.rounded_compare(mean, 10160)) + + def test_agg_local_mean(self): + from pyspark.sql import Row + from pyrasterframes.rf_types import Tile + import numpy as np + + # this is really testing the nodata propagation in the agg local summation + ct = CellType.int8().with_no_data_value(4) + df = self.spark.createDataFrame([ + Row(tile=Tile(np.array([[1, 2, 3, 4, 5, 6]]), ct)), + Row(tile=Tile(np.array([[1, 2, 4, 3, 5, 6]]), ct)), + ]) + + result = df.agg(rf_agg_local_mean('tile').alias('mean')).first().mean + + expected = Tile(np.array([[1.0, 2.0, 3.0, 3.0, 5.0, 6.0]]), CellType.float64()) + self.assertEqual(result, expected) + + def test_aggregations(self): + aggs = self.rf.agg( + rf_agg_data_cells('tile'), + rf_agg_no_data_cells('tile'), + rf_agg_stats('tile'), + rf_agg_approx_histogram('tile') + ) + row = aggs.first() + + # print(row['rf_agg_data_cells(tile)']) + self.assertEqual(row['rf_agg_data_cells(tile)'], 387000) + self.assertEqual(row['rf_agg_no_data_cells(tile)'], 1000) + self.assertEqual(row['rf_agg_stats(tile)'].data_cells, row['rf_agg_data_cells(tile)']) + + def test_sql(self): + + self.rf.createOrReplaceTempView("rf_test_sql") + + arith = self.spark.sql("""SELECT tile, + rf_local_add(tile, 1) AS add_one, + rf_local_subtract(tile, 1) AS less_one, + rf_local_multiply(tile, 2) AS times_two, + rf_local_divide( + rf_convert_cell_type(tile, "float32"), + 2) AS over_two + FROM rf_test_sql""") + + arith.createOrReplaceTempView('rf_test_sql_1') + arith.show(truncate=False) + stats = self.spark.sql(""" + SELECT rf_tile_mean(tile) as base, + rf_tile_mean(add_one) as plus_one, + rf_tile_mean(less_one) as minus_one, + rf_tile_mean(times_two) as double, + rf_tile_mean(over_two) as half, + rf_no_data_cells(tile) as nd + + FROM rf_test_sql_1 + ORDER BY rf_no_data_cells(tile) + """) + stats.show(truncate=False) + stats.createOrReplaceTempView('rf_test_sql_stats') + + compare = self.spark.sql(""" + SELECT + plus_one - 1.0 = base as add, + minus_one + 1.0 = base as subtract, + double / 2.0 = base as multiply, + half * 2.0 = base as divide, + nd + FROM rf_test_sql_stats + """) + + expect_row1 = compare.orderBy('nd').first() + + self.assertTrue(expect_row1.subtract) + self.assertTrue(expect_row1.multiply) + self.assertTrue(expect_row1.divide) + self.assertEqual(expect_row1.nd, 0) + self.assertTrue(expect_row1.add) + + expect_row2 = compare.orderBy('nd', ascending=False).first() + + self.assertTrue(expect_row2.subtract) + self.assertTrue(expect_row2.multiply) + self.assertTrue(expect_row2.divide) + self.assertTrue(expect_row2.nd > 0) + self.assertTrue(expect_row2.add) # <-- Would fail in a case where ND + 1 = 1 + + def test_explode(self): + import pyspark.sql.functions as F + self.rf.select('spatial_key', rf_explode_tiles('tile')).show() + # +-----------+------------+---------+-------+ + # |spatial_key|column_index|row_index|tile | + # +-----------+------------+---------+-------+ + # |[2,1] |4 |0 |10150.0| + cell = self.rf.select(self.rf.spatial_key_column(), rf_explode_tiles(self.rf.tile)) \ + .where(F.col("spatial_key.col") == 2) \ + .where(F.col("spatial_key.row") == 1) \ + .where(F.col("column_index") == 4) \ + .where(F.col("row_index") == 0) \ + .select(F.col("tile")) \ + .collect()[0][0] + self.assertEqual(cell, 10150.0) + + # Test the sample version + frac = 0.01 + sample_count = self.rf.select(rf_explode_tiles_sample(frac, 1872, 'tile')).count() + print('Sample count is {}'.format(sample_count)) + self.assertTrue(sample_count > 0) + self.assertTrue(sample_count < (frac * 1.1) * 387000) # give some wiggle room + + def test_mask_by_value(self): + from pyspark.sql.functions import lit + + # create an artificial mask for values > 25000; masking value will be 4 + mask_value = 4 + + rf1 = self.rf.select(self.rf.tile, + rf_local_multiply( + rf_convert_cell_type( + rf_local_greater_int(self.rf.tile, 25000), + "uint8"), + lit(mask_value)).alias('mask')) + rf2 = rf1.select(rf1.tile, rf_mask_by_value(rf1.tile, rf1.mask, lit(mask_value)).alias('masked')) + result = rf2.agg(rf_agg_no_data_cells(rf2.tile) < rf_agg_no_data_cells(rf2.masked)) \ + .collect()[0][0] + self.assertTrue(result) + + rf3 = rf1.select(rf1.tile, rf_inverse_mask_by_value(rf1.tile, rf1.mask, lit(mask_value)).alias('masked')) + result = rf3.agg(rf_agg_no_data_cells(rf3.tile) < rf_agg_no_data_cells(rf3.masked)) \ + .collect()[0][0] + self.assertTrue(result) + + def test_resample(self): + from pyspark.sql.functions import lit + result = self.rf.select( + rf_tile_min(rf_local_equal( + rf_resample(rf_resample(self.rf.tile, lit(2)), lit(0.5)), + self.rf.tile)) + ).collect()[0][0] + + self.assertTrue(result == 1) # short hand for all values are true + + def test_exists_for_all(self): + df = self.rf.withColumn('should_exist', rf_make_ones_tile(5, 5, 'int8')) \ + .withColumn('should_not_exist', rf_make_zeros_tile(5, 5, 'int8')) + + should_exist = df.select(rf_exists(df.should_exist).alias('se')).take(1)[0].se + self.assertTrue(should_exist) + + should_not_exist = df.select(rf_exists(df.should_not_exist).alias('se')).take(1)[0].se + self.assertTrue(not should_not_exist) + + self.assertTrue(df.select(rf_for_all(df.should_exist).alias('se')).take(1)[0].se) + self.assertTrue(not df.select(rf_for_all(df.should_not_exist).alias('se')).take(1)[0].se) + + def test_cell_type_in_functions(self): + from pyrasterframes.rf_types import CellType + ct = CellType.float32() + ct_str = CellType.float32().with_no_data_value(-999).cell_type_name # NB int ND into float celltype + + df = self.rf.withColumn('ct_str', rf_convert_cell_type('tile', ct_str)) \ + .withColumn('ct', rf_convert_cell_type('tile', ct)) \ + .withColumn('make', rf_make_constant_tile(99, 3, 4, CellType.int8())) + + result = df.select('ct', 'ct_str', 'make').first() + + self.assertEqual(result['ct'].cell_type, ct) + self.assertEqual(result['ct_str'].cell_type, CellType(ct_str)) + self.assertEqual(result['make'].cell_type, CellType.int8()) diff --git a/pyrasterframes/src/main/python/tests/VectorTypesTests.py b/pyrasterframes/src/main/python/tests/VectorTypesTests.py new file mode 100644 index 000000000..e31f26b43 --- /dev/null +++ b/pyrasterframes/src/main/python/tests/VectorTypesTests.py @@ -0,0 +1,158 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from pyrasterframes.rasterfunctions import * +from pyspark.sql import Row +from pyspark.sql.functions import * + +from . import TestEnvironment + + +class VectorTypes(TestEnvironment): + + def setUp(self): + self.create_layer() + import pandas as pd + self.pandas_df = pd.DataFrame({ + 'eye': ['a', 'b', 'c', 'd'], + 'x': [0.0, 1.0, 2.0, 3.0], + 'y': [-4.0, -3.0, -2.0, -1.0], + }) + df = self.spark.createDataFrame(self.pandas_df) + df = df.withColumn("point_geom", + st_point(df.x, df.y) + ) + self.df = df.withColumn("poly_geom", st_bufferPoint(df.point_geom, lit(1250.0))) + + def test_spatial_relations(self): + from pyspark.sql.functions import udf, sum + from geomesa_pyspark.types import PointUDT + import shapely + import numpy.testing + + # Use python shapely UDT in a UDF + @udf("double") + def area_fn(g): + return g.area + + @udf("double") + def length_fn(g): + return g.length + + df = self.df.withColumn("poly_area", area_fn(self.df.poly_geom)) + df = df.withColumn("poly_len", length_fn(df.poly_geom)) + + # Return UDT in a UDF! + def some_point(g): + return g.representative_point() + + some_point_udf = udf(some_point, PointUDT()) + + df = df.withColumn("any_point", some_point_udf(df.poly_geom)) + # spark-side UDF/UDT are correct + intersect_total = df.agg(sum( + st_intersects(df.poly_geom, df.any_point).astype('double') + ).alias('s')).collect()[0].s + self.assertTrue(intersect_total == df.count()) + + # Collect to python driver in shapely UDT + pandas_df_out = df.toPandas() + + # Confirm we get a shapely type back from st_* function and UDF + self.assertIsInstance(pandas_df_out.poly_geom.iloc[0], shapely.geometry.Polygon) + self.assertIsInstance(pandas_df_out.any_point.iloc[0], shapely.geometry.Point) + + # And our spark-side manipulations were correct + xs_correct = pandas_df_out.point_geom.apply(lambda g: g.coords[0][0]) == self.pandas_df.x + self.assertTrue(all(xs_correct)) + + centroid_ys = pandas_df_out.poly_geom.apply(lambda g: + g.centroid.coords[0][1]).tolist() + numpy.testing.assert_almost_equal(centroid_ys, self.pandas_df.y.tolist()) + + # Including from UDF's + numpy.testing.assert_almost_equal( + pandas_df_out.poly_geom.apply(lambda g: g.area).values, + pandas_df_out.poly_area.values + ) + numpy.testing.assert_almost_equal( + pandas_df_out.poly_geom.apply(lambda g: g.length).values, + pandas_df_out.poly_len.values + ) + + def test_geometry_udf(self): + from geomesa_pyspark.types import PolygonUDT + # simple test that raster contents are not invalid + + # create a udf to buffer (the bounds) polygon + def _buffer(g, d): + return g.buffer(d) + + @udf("double") + def area(g): + return g.area + + buffer_udf = udf(_buffer, PolygonUDT()) + + buf_cells = 10 + with_poly = self.rf.withColumn('poly', buffer_udf(self.rf.geometry, lit(-15 * buf_cells))) # cell res is 15x15 + area = with_poly.select(area('poly') < area('geometry')) + area_result = area.collect() + self.assertTrue(all([r[0] for r in area_result])) + + def test_rasterize(self): + from geomesa_pyspark.types import PolygonUDT + + @udf(PolygonUDT()) + def buffer(g, d): + return g.buffer(d) + + # start with known polygon, the tile extents, **negative buffered** by 10 cells + buf_cells = 10 + with_poly = self.rf.withColumn('poly', buffer(self.rf.geometry, lit(-15 * buf_cells))) # cell res is 15x15 + + # rasterize value 16 into buffer shape. + cols = 194 # from dims of tile + rows = 250 # from dims of tile + with_raster = with_poly.withColumn('rasterized', + rf_rasterize('poly', 'geometry', lit(16), lit(cols), lit(rows))) + result = with_raster.select(rf_tile_sum(rf_local_equal_int(with_raster.rasterized, 16)), + rf_tile_sum(with_raster.rasterized)) + # + expected_burned_in_cells = (cols - 2 * buf_cells) * (rows - 2 * buf_cells) + self.assertEqual(result.first()[0], float(expected_burned_in_cells)) + self.assertEqual(result.first()[1], 16. * expected_burned_in_cells) + + def test_parse_crs(self): + df = self.spark.createDataFrame([Row(id=1)]) + self.assertEqual(df.select(rf_mk_crs('EPSG:4326')).count(), 1) + + def test_reproject(self): + reprojected = self.rf.withColumn('reprojected', + st_reproject('center', rf_mk_crs('EPSG:4326'), rf_mk_crs('EPSG:3857'))) + reprojected.show() + self.assertEqual(reprojected.count(), 8) + + def test_geojson(self): + import os + sample = 'file://' + os.path.join(self.resource_dir, 'buildings.geojson') + geo = self.spark.read.geojson(sample) + geo.show() + self.assertEqual(geo.select('geometry').count(), 8) diff --git a/pyrasterframes/src/main/python/tests/__init__.py b/pyrasterframes/src/main/python/tests/__init__.py new file mode 100644 index 000000000..177c9a8c7 --- /dev/null +++ b/pyrasterframes/src/main/python/tests/__init__.py @@ -0,0 +1,87 @@ +# +# This software is licensed under the Apache 2 license, quoted below. +# +# Copyright 2019 Astraea, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# [http://www.apache.org/licenses/LICENSE-2.0] +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import glob +import os +import unittest + +from pyrasterframes.utils import create_rf_spark_session + +import sys + +if sys.version_info[0] > 2: + import builtins +else: + import __builtin__ as builtins + + +def resource_dir(): + def pdir(curr): + return os.path.dirname(curr) + + here = os.path.dirname(os.path.realpath(__file__)) + scala_target = os.path.realpath(os.path.join(pdir(pdir(here)), 'scala-2.11')) + rez_dir = os.path.realpath(os.path.join(scala_target, 'test-classes')) + # If not running in build mode, try source dirs. + if not os.path.exists(rez_dir): + rez_dir = os.path.realpath(os.path.join(pdir(pdir(pdir(here))), 'test', 'resources')) + return rez_dir + + +def spark_test_session(): + spark = create_rf_spark_session() + spark.sparkContext.setLogLevel('ERROR') + + print("Spark Version: " + spark.version) + print("Spark Config: " + str(spark.sparkContext._conf.getAll())) + + return spark + + +class TestEnvironment(unittest.TestCase): + """ + Base class for tests. + """ + + def rounded_compare(self, val1, val2): + print('Comparing {} and {} using round()'.format(val1, val2)) + return builtins.round(val1) == builtins.round(val2) + + @classmethod + def setUpClass(cls): + # hard-coded relative path for resources + cls.resource_dir = resource_dir() + + cls.spark = spark_test_session() + + cls.img_uri = 'file://' + os.path.join(cls.resource_dir, 'L8-B8-Robinson-IL.tiff') + + def create_layer(self): + from pyrasterframes.rasterfunctions import rf_convert_cell_type + # load something into a rasterframe + rf = self.spark.read.geotiff(self.img_uri) \ + .with_bounds() \ + .with_center() + + # convert the tile cell type to provide for other operations + self.rf = rf.withColumn('tile2', rf_convert_cell_type('tile', 'float32')) \ + .drop('tile') \ + .withColumnRenamed('tile2', 'tile').as_layer() + # cls.rf.show() diff --git a/pyrasterframes/src/main/python/tests/coverage-report.sh b/pyrasterframes/src/main/python/tests/coverage-report.sh new file mode 100755 index 000000000..6b547e026 --- /dev/null +++ b/pyrasterframes/src/main/python/tests/coverage-report.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash -e + +# If `coverage` tool isn't installed: `{pip|conda} install coverage` + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +cd "$( dirname "${BASH_SOURCE[0]}" )"/.. + +coverage run setup.py test && coverage html --omit='.eggs/*,setup.py' && open htmlcov/index.html \ No newline at end of file diff --git a/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala deleted file mode 100644 index b80a8a3f8..000000000 --- a/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala +++ /dev/null @@ -1,202 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017-2018 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package astraea.spark.rasterframes.py - -import astraea.spark.rasterframes._ -import astraea.spark.rasterframes.util.CRSParser -import com.vividsolutions.jts.geom.Geometry -import geotrellis.raster.{ArrayTile, CellType, MultibandTile} -import geotrellis.spark.io._ -import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD, SpaceTimeKey, SpatialKey, TileLayerMetadata} -import org.apache.spark.sql._ -import org.locationtech.geomesa.spark.jts.util.WKBUtils -import spray.json._ - -import scala.collection.JavaConverters._ - -/** - * py4j access wrapper to RasterFrame entry points. - * - * @since 11/6/17 - */ -class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions - with org.locationtech.geomesa.spark.jts.DataFrameFunctions.Library { - - sparkSession.withRasterFrames - - def toSpatialMultibandTileLayerRDD(rf: RasterFrame): MultibandTileLayerRDD[SpatialKey] = - rf.toMultibandTileLayerRDD match { - case Left(spatial) => spatial - case Right(other) => throw new Exception(s"Expected a MultibandTileLayerRDD[SpatailKey] but got $other instead") - } - - def toSpaceTimeMultibandTileLayerRDD(rf: RasterFrame): MultibandTileLayerRDD[SpaceTimeKey] = - rf.toMultibandTileLayerRDD match { - case Right(temporal) => temporal - case Left(other) => throw new Exception(s"Expected a MultibandTileLayerRDD[SpaceTimeKey] but got $other instead") - } - - /** - * Converts a `ContextRDD[Spatialkey, MultibandTile, TileLayerMedadata[Spatialkey]]` to a RasterFrame - */ - def asRF( - layer: ContextRDD[SpatialKey, MultibandTile, TileLayerMetadata[SpatialKey]], - bandCount: java.lang.Integer - ): RasterFrame = { - implicit val pr = PairRDDConverter.forSpatialMultiband(bandCount.toInt) - layer.toRF - } - - /** - * Converts a `ContextRDD[SpaceTimeKey, MultibandTile, TileLayerMedadata[SpaceTimeKey]]` to a RasterFrame - */ - def asRF( - layer: ContextRDD[SpaceTimeKey, MultibandTile, TileLayerMetadata[SpaceTimeKey]], - bandCount: java.lang.Integer - )(implicit d: DummyImplicit): RasterFrame = { - implicit val pr = PairRDDConverter.forSpaceTimeMultiband(bandCount.toInt) - layer.toRF - } - - /** - * Base conversion to RasterFrame - */ - def asRF(df: DataFrame): RasterFrame = { - df.asRF - } - - /** - * Conversion to RasterFrame with spatial key column and TileLayerMetadata specified. - */ - def asRF(df: DataFrame, spatialKey: Column, tlm: String): RasterFrame = { - val jtlm = tlm.parseJson.convertTo[TileLayerMetadata[SpatialKey]] - df.asRF(spatialKey, jtlm) - } - - /** - * Convenience functions for use in Python - */ - def cell_type(name: String): CellType = CellType.fromName(name) - - /** - * Convenience list of valid cell type strings - * @return Java List of String, which py4j can interpret as a python `list` - */ - def cell_types = { - astraea.spark.rasterframes.functions.cellTypes().asJava - } - - /** DESERIALIZATION **/ - - def generate_tile(cellType: String, cols: Int, rows: Int, bytes: Array[Byte]): ArrayTile = { - ArrayTile.fromBytes(bytes, this.cell_type(cellType), cols, rows) - } - - def generate_geometry(obj: Array[Byte]): Geometry = WKBUtils.read(obj) - - def explode_tiles_sample(sampleFraction: Double, seed: Long, cols: Column*): Column = - explode_tiles_sample(sampleFraction, Some(seed), cols: _*) - - - def tileColumns(df: DataFrame): Array[Column] = - df.asRF.tileColumns.toArray - - def spatialKeyColumn(df: DataFrame): Column = - df.asRF.spatialKeyColumn - - def temporalKeyColumn(df: DataFrame): Column = - df.asRF.temporalKeyColumn.orNull - - def tile_to_int_array(col: Column): Column = tile_to_array_int(col) - - def tile_to_double_array(col: Column): Column = tile_to_array_double(col) - - // All the scalar tile arithmetic functions - - def local_add_scalar(col: Column, scalar: Double): Column = local_add[Double](col, scalar) - - def local_add_scalar_int(col: Column, scalar: Int): Column = local_add[Int](col, scalar) - - def local_subtract_scalar(col: Column, scalar: Double): Column = local_subtract[Double](col, scalar) - - def local_subtract_scalar_int(col: Column, scalar: Int): Column = local_subtract[Int](col, scalar) - - def local_divide_scalar(col: Column, scalar: Double): Column = local_divide[Double](col, scalar) - - def local_divide_scalar_int(col: Column, scalar: Int): Column = local_divide[Int](col, scalar) - - def local_multiply_scalar(col: Column, scalar: Double): Column = local_multiply[Double](col, scalar) - - def local_multiply_scalar_int(col: Column, scalar: Int): Column = local_multiply[Int](col, scalar) - - def local_less_scalar(col: Column, scalar: Double): Column = local_less[Double](col, scalar) - - def local_less_scalar_int(col: Column, scalar: Int): Column = local_less[Int](col, scalar) - - def local_less_equal_scalar(col: Column, scalar: Double): Column = local_less_equal[Double](col, scalar) - - def local_less_equal_scalar_int(col: Column, scalar: Int): Column = local_less_equal[Int](col, scalar) - - def local_greater_scalar(col: Column, scalar: Double): Column = local_greater[Double](col, scalar) - - def local_greater_scalar_int(col: Column, scalar: Int): Column = local_greater[Int](col, scalar) - - def local_greater_equal_scalar(col: Column, scalar: Double): Column = local_greater_equal[Double](col, scalar) - - def local_greater_equal_scalar_int(col: Column, scalar: Int): Column = local_greater_equal[Int](col, scalar) - - def local_equal_scalar(col: Column, scalar: Double): Column = local_equal[Double](col, scalar) - - def local_equal_scalar_int(col: Column, scalar: Int): Column = local_equal[Int](col, scalar) - - def local_unequal_scalar(col: Column, scalar: Double): Column = local_unequal[Double](col, scalar) - - def local_unequal_scalar_int(col: Column, scalar: Int): Column = local_unequal[Int](col, scalar) - - // return toRaster, get just the tile, and make an array out of it - def toIntRaster(df: DataFrame, colname: String, cols: Int, rows: Int): Array[Int] = { - df.asRF.toRaster(df.col(colname), cols, rows).toArray() - } - - def toDoubleRaster(df: DataFrame, colname: String, cols: Int, rows: Int): Array[Double] = { - df.asRF.toRaster(df.col(colname), cols, rows).toArrayDouble() - } - - def tileLayerMetadata(df: DataFrame): String = - // The `fold` is required because an `Either` is retured, depending on the key type. - df.asRF.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint - - def spatialJoin(df: DataFrame, right: DataFrame): RasterFrame = df.asRF.spatialJoin(right.asRF) - - def withBounds(df: DataFrame): RasterFrame = df.asRF.withBounds() - - def withCenter(df: DataFrame): RasterFrame = df.asRF.withCenter() - - def withCenterLatLng(df: DataFrame): RasterFrame = df.asRF.withCenterLatLng() - - def withSpatialIndex(df: DataFrame): RasterFrame = df.asRF.withSpatialIndex() - - def reproject_geometry(geometryCol: Column, srcName: String, dstName: String): Column = { - val src = CRSParser(srcName) - val dst = CRSParser(dstName) - reproject_geometry(geometryCol, src, dst) - } - - def listToSeq(cols: java.util.ArrayList[AnyRef]): Seq[AnyRef] = cols.asScala -} diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala new file mode 100644 index 000000000..f950cf2f3 --- /dev/null +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -0,0 +1,239 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2017-2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.locationtech.rasterframes.py + +import java.nio.ByteBuffer + +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, MultibandTile} +import geotrellis.spark.io._ +import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD, SpaceTimeKey, SpatialKey, TileLayerMetadata} +import geotrellis.vector.Extent +import org.apache.spark.sql._ +import org.locationtech.rasterframes +import org.locationtech.rasterframes.extensions.RasterJoin +import org.locationtech.rasterframes.model.LazyCRS +import org.locationtech.rasterframes.ref.{GDALRasterSource, RasterRef, RasterSource} +import org.locationtech.rasterframes.util.KryoSupport +import org.locationtech.rasterframes.{RasterFunctions, _} +import spray.json._ + +import scala.collection.JavaConverters._ + +/** + * py4j access wrapper to RasterFrameLayer entry points. + * + * @since 11/6/17 + */ +class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions + with org.locationtech.geomesa.spark.jts.DataFrameFunctions.Library { + + sparkSession.withRasterFrames + + def buildInfo(): java.util.HashMap[String, String] = { + val retval = new java.util.HashMap[String, String]() + RFBuildInfo.toMap.foreach { + case (k, v) => retval.put(k, String.valueOf(v)) + } + retval.put("GDAL", GDALRasterSource.gdalVersion()) + retval + } + + def toSpatialMultibandTileLayerRDD(rf: RasterFrameLayer): MultibandTileLayerRDD[SpatialKey] = + rf.toMultibandTileLayerRDD match { + case Left(spatial) => spatial + case Right(other) => throw new Exception(s"Expected a MultibandTileLayerRDD[SpatailKey] but got $other instead") + } + + def toSpaceTimeMultibandTileLayerRDD(rf: RasterFrameLayer): MultibandTileLayerRDD[SpaceTimeKey] = + rf.toMultibandTileLayerRDD match { + case Right(temporal) => temporal + case Left(other) => throw new Exception(s"Expected a MultibandTileLayerRDD[SpaceTimeKey] but got $other instead") + } + + /** + * Converts a `ContextRDD[Spatialkey, MultibandTile, TileLayerMedadata[Spatialkey]]` to a RasterFrameLayer + */ + def asLayer( + layer: ContextRDD[SpatialKey, MultibandTile, TileLayerMetadata[SpatialKey]], + bandCount: java.lang.Integer + ): RasterFrameLayer = { + implicit val pr = PairRDDConverter.forSpatialMultiband(bandCount.toInt) + layer.toLayer + } + + /** + * Converts a `ContextRDD[SpaceTimeKey, MultibandTile, TileLayerMedadata[SpaceTimeKey]]` to a RasterFrameLayer + */ + def asLayer( + layer: ContextRDD[SpaceTimeKey, MultibandTile, TileLayerMetadata[SpaceTimeKey]], + bandCount: java.lang.Integer + )(implicit d: DummyImplicit): RasterFrameLayer = { + implicit val pr = PairRDDConverter.forSpaceTimeMultiband(bandCount.toInt) + layer.toLayer + } + + /** + * Base conversion to RasterFrameLayer + */ + def asLayer(df: DataFrame): RasterFrameLayer = { + df.asLayer + } + + /** + * Conversion to RasterFrameLayer with spatial key column and TileLayerMetadata specified. + */ + def asLayer(df: DataFrame, spatialKey: Column, tlm: String): RasterFrameLayer = { + val jtlm = tlm.parseJson.convertTo[TileLayerMetadata[SpatialKey]] + df.asLayer(spatialKey, jtlm) + } + + /** + * Left spatial join managing reprojection and merging of `other` + */ + def rasterJoin(df: DataFrame, other: DataFrame): DataFrame = RasterJoin(df, other) + + /** + * Left spatial join managing reprojection and merging of `other`; uses extent and CRS columns to determine if rows intersect + */ + def rasterJoin(df: DataFrame, other: DataFrame, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = + RasterJoin(df, other, leftExtent, leftCRS, rightExtent, rightCRS) + + /** + * Left spatial join managing reprojection and merging of `other`; uses joinExprs to conduct initial join then extent and CRS columns to determine if rows intersect + */ + def rasterJoin(df: DataFrame, other: DataFrame, joinExprs: Column, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column): DataFrame = + RasterJoin(df, other, joinExprs, leftExtent, leftCRS, rightExtent, rightCRS) + + + /** + * Convenience functions for use in Python + */ + def _parse_cell_type(name: String): CellType = CellType.fromName(name) + + /** + * Convenience list of valid cell type strings + * @return Java List of String, which py4j can interpret as a python `list` + */ + def rf_cell_types = { + org.locationtech.rasterframes.functions.cellTypes().asJava + } + + def rf_explode_tiles_sample(sampleFraction: Double, seed: Long, cols: Column*): Column = + rf_explode_tiles_sample(sampleFraction, Some(seed), cols: _*) + + def tileColumns(df: DataFrame): Array[Column] = + df.asLayer.tileColumns.toArray + + def spatialKeyColumn(df: DataFrame): Column = + df.asLayer.spatialKeyColumn + + def temporalKeyColumn(df: DataFrame): Column = + df.asLayer.temporalKeyColumn.orNull + + // All the scalar tile arithmetic functions + + def rf_local_add_double(col: Column, scalar: Double): Column = rf_local_add[Double](col, scalar) + + def rf_local_add_int(col: Column, scalar: Int): Column = rf_local_add[Int](col, scalar) + + def rf_local_subtract_double(col: Column, scalar: Double): Column = rf_local_subtract[Double](col, scalar) + + def rf_local_subtract_int(col: Column, scalar: Int): Column = rf_local_subtract[Int](col, scalar) + + def rf_local_divide_double(col: Column, scalar: Double): Column = rf_local_divide[Double](col, scalar) + + def rf_local_divide_int(col: Column, scalar: Int): Column = rf_local_divide[Int](col, scalar) + + def rf_local_multiply_double(col: Column, scalar: Double): Column = rf_local_multiply[Double](col, scalar) + + def rf_local_multiply_int(col: Column, scalar: Int): Column = rf_local_multiply[Int](col, scalar) + + def rf_local_less_double(col: Column, scalar: Double): Column = rf_local_less[Double](col, scalar) + + def rf_local_less_int(col: Column, scalar: Int): Column = rf_local_less[Int](col, scalar) + + def rf_local_less_equal_double(col: Column, scalar: Double): Column = rf_local_less_equal[Double](col, scalar) + + def rf_local_less_equal_int(col: Column, scalar: Int): Column = rf_local_less_equal[Int](col, scalar) + + def rf_local_greater_double(col: Column, scalar: Double): Column = rf_local_greater[Double](col, scalar) + + def rf_local_greater_int(col: Column, scalar: Int): Column = rf_local_greater[Int](col, scalar) + + def rf_local_greater_equal_double(col: Column, scalar: Double): Column = rf_local_greater_equal[Double](col, scalar) + + def rf_local_greater_equal_int(col: Column, scalar: Int): Column = rf_local_greater_equal[Int](col, scalar) + + def rf_local_equal_double(col: Column, scalar: Double): Column = rf_local_equal[Double](col, scalar) + + def rf_local_equal_int(col: Column, scalar: Int): Column = rf_local_equal[Int](col, scalar) + + def rf_local_unequal_double(col: Column, scalar: Double): Column = rf_local_unequal[Double](col, scalar) + + def rf_local_unequal_int(col: Column, scalar: Int): Column = rf_local_unequal[Int](col, scalar) + + def _make_crs_literal(crsText: String): Column = { + rasterframes.encoders.serialized_literal[CRS](LazyCRS(crsText)) + } + + // return toRaster, get just the tile, and make an array out of it + def toIntRaster(df: DataFrame, colname: String, cols: Int, rows: Int): Array[Int] = { + df.asLayer.toRaster(df.col(colname), cols, rows).tile.toArray() + } + + def toDoubleRaster(df: DataFrame, colname: String, cols: Int, rows: Int): Array[Double] = { + df.asLayer.toRaster(df.col(colname), cols, rows).tile.toArrayDouble() + } + + def tileLayerMetadata(df: DataFrame): String = + // The `fold` is required because an `Either` is retured, depending on the key type. + df.asLayer.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint + + def spatialJoin(df: DataFrame, right: DataFrame): RasterFrameLayer = df.asLayer.spatialJoin(right.asLayer) + + def withBounds(df: DataFrame): RasterFrameLayer = df.asLayer.withGeometry() + + def withCenter(df: DataFrame): RasterFrameLayer = df.asLayer.withCenter() + + def withCenterLatLng(df: DataFrame): RasterFrameLayer = df.asLayer.withCenterLatLng() + + def withSpatialIndex(df: DataFrame): RasterFrameLayer = df.asLayer.withSpatialIndex() + + //----------------------------Support Routines----------------------------------------- + + def _listToSeq(cols: java.util.ArrayList[AnyRef]): Seq[AnyRef] = cols.asScala + + type jInt = java.lang.Integer + type jDouble = java.lang.Double + // NB: Tightly coupled to the `RFContext.resolve_raster_ref` method in `pyrasterframes.context`. */ + def _resolveRasterRef(srcBin: Array[Byte], bandIndex: jInt, xmin: jDouble, ymin: jDouble, xmax: jDouble, ymax: jDouble): AnyRef = { + val src = KryoSupport.deserialize[RasterSource](ByteBuffer.wrap(srcBin)) + val extent = Extent(xmin, ymin, xmax, ymax) + val ref = RasterRef(src, bandIndex, Some(extent)) + ref.tile.toArrayTile().toBytes() + } + + def _dfToMarkdown(df: DataFrame, numRows: Int, truncate: Boolean): String = { + import rasterframes.util.DFWithPrettyPrint + df.toMarkdown(numRows, truncate) + } +} diff --git a/deployment/docker/jupyter/samples/L8-B1-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B1-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B1-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B1-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B10-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B10-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B10-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B10-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B11-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B11-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B11-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B11-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B2-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B2-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B2-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B2-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B3-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B3-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B3-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B3-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B4-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B4-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B4-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B4-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B5-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B5-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B5-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B5-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B6-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B6-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B6-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B6-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B7-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B7-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B7-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B7-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B8-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B8-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B8-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B8-Elkton-VA.tiff diff --git a/deployment/docker/jupyter/samples/L8-B8-Robinson-IL.tiff b/pyrasterframes/src/test/resources/L8-B8-Robinson-IL.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B8-Robinson-IL.tiff rename to pyrasterframes/src/test/resources/L8-B8-Robinson-IL.tiff diff --git a/deployment/docker/jupyter/samples/L8-B9-Elkton-VA.tiff b/pyrasterframes/src/test/resources/L8-B9-Elkton-VA.tiff similarity index 100% rename from deployment/docker/jupyter/samples/L8-B9-Elkton-VA.tiff rename to pyrasterframes/src/test/resources/L8-B9-Elkton-VA.tiff diff --git a/pyrasterframes/src/test/resources/buildings.geojson b/pyrasterframes/src/test/resources/buildings.geojson new file mode 100644 index 000000000..91aea5924 --- /dev/null +++ b/pyrasterframes/src/test/resources/buildings.geojson @@ -0,0 +1,899 @@ +{ + "type": "FeatureCollection", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG::3968" + } + }, + "bbox": [ + 10453.2340000011, + 137465.4443, + 0, + 203572.226800002, + 261914.089200001, + 518 + ], + "features": [ + { + "type": "Feature", + "properties": { + "OBJECTID": 2560367, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23712149512", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 112.01513810500001, + "SHAPE_Area": 612.03868604499996, + "AREA_SQFT": 6587.9 + }, + "bbox": [ + 23695.017599999904633, + 149496.648900002241135, + 0.0, + 23732.747999999672174, + 149527.1985, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23730.124099999666214, + 149514.097699999809265, + 0.0 + ], + [ + 23728.569000001996756, + 149511.027800001204014, + 0.0 + ], + [ + 23732.747999999672174, + 149508.911100000143051, + 0.0 + ], + [ + 23731.134500000625849, + 149505.72520000115037, + 0.0 + ], + [ + 23727.252300001680851, + 149507.691500000655651, + 0.0 + ], + [ + 23725.473999999463558, + 149504.180500000715256, + 0.0 + ], + [ + 23722.397100001573563, + 149505.523900002241135, + 0.0 + ], + [ + 23717.634100001305342, + 149496.648900002241135, + 0.0 + ], + [ + 23696.2179000005126, + 149507.229299999773502, + 0.0 + ], + [ + 23698.083399999886751, + 149510.912300001829863, + 0.0 + ], + [ + 23696.541700001806021, + 149511.693300001323223, + 0.0 + ], + [ + 23695.017599999904633, + 149512.464999999850988, + 0.0 + ], + [ + 23702.387800000607967, + 149527.1985, + 0.0 + ], + [ + 23730.124099999666214, + 149514.097699999809265, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2561065, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "22784149837", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 140.080104933, + "SHAPE_Area": 572.44039797000005, + "AREA_SQFT": 6161.67 + }, + "bbox": [ + 22778.690500002354383, + 149807.139500003308058, + 0.0, + 22788.61260000243783, + 149867.804400000721216, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 22788.599300000816584, + 149866.041400000452995, + 0.0 + ], + [ + 22788.586899999529123, + 149864.452200002968311, + 0.0 + ], + [ + 22788.545500002801418, + 149859.174200002104044, + 0.0 + ], + [ + 22788.532500002533197, + 149857.504900000989437, + 0.0 + ], + [ + 22788.493900001049042, + 149852.586800001561642, + 0.0 + ], + [ + 22788.480700001120567, + 149850.917000003159046, + 0.0 + ], + [ + 22788.44030000269413, + 149845.7555, + 0.0 + ], + [ + 22788.427200000733137, + 149844.088899999856949, + 0.0 + ], + [ + 22788.36600000038743, + 149836.274000000208616, + 0.0 + ], + [ + 22788.355200000107288, + 149834.883799999952, + 0.0 + ], + [ + 22788.314600002020597, + 149829.721200000494719, + 0.0 + ], + [ + 22788.302200000733137, + 149828.140100002288818, + 0.0 + ], + [ + 22788.262099999934435, + 149823.015900000929832, + 0.0 + ], + [ + 22788.249400001019239, + 149821.38910000026226, + 0.0 + ], + [ + 22788.20890000090003, + 149816.230100002139807, + 0.0 + ], + [ + 22788.19649999961257, + 149814.641700003296137, + 0.0 + ], + [ + 22788.157099999487, + 149809.631100002676249, + 0.0 + ], + [ + 22788.143800001591444, + 149807.931900002062321, + 0.0 + ], + [ + 22788.137600000947714, + 149807.139500003308058, + 0.0 + ], + [ + 22778.690500002354383, + 149807.213600002229214, + 0.0 + ], + [ + 22778.701400000602007, + 149808.6064000017941, + 0.0 + ], + [ + 22778.724300000816584, + 149811.520500000566244, + 0.0 + ], + [ + 22778.757200002670288, + 149815.718500003218651, + 0.0 + ], + [ + 22778.776700001209974, + 149818.198900002986193, + 0.0 + ], + [ + 22778.80800000205636, + 149822.202100001275539, + 0.0 + ], + [ + 22778.830000001937151, + 149825.011500000953674, + 0.0 + ], + [ + 22778.861900001764297, + 149829.083700001239777, + 0.0 + ], + [ + 22778.882699999958277, + 149831.727000001817942, + 0.0 + ], + [ + 22778.913499999791384, + 149835.65990000218153, + 0.0 + ], + [ + 22778.934599999338388, + 149838.344300001859665, + 0.0 + ], + [ + 22778.967199999839067, + 149842.509800001978874, + 0.0 + ], + [ + 22778.99040000140667, + 149845.455000001937151, + 0.0 + ], + [ + 22779.019000001251698, + 149849.091400001198053, + 0.0 + ], + [ + 22779.041999999433756, + 149852.034200001507998, + 0.0 + ], + [ + 22779.073300000280142, + 149856.03490000218153, + 0.0 + ], + [ + 22779.094000000506639, + 149858.6824000030756, + 0.0 + ], + [ + 22779.124700002372265, + 149862.587100002914667, + 0.0 + ], + [ + 22779.149300001561642, + 149865.729600001126528, + 0.0 + ], + [ + 22779.159200001507998, + 149866.985300000756979, + 0.0 + ], + [ + 22779.165600001811981, + 149867.804400000721216, + 0.0 + ], + [ + 22788.61260000243783, + 149867.730399999767542, + 0.0 + ], + [ + 22788.599300000816584, + 149866.041400000452995, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2564971, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "22828150049", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 90.962539370399995, + "SHAPE_Area": 469.83376775699998, + "AREA_SQFT": 5057.23 + }, + "bbox": [ + 22814.135000001639128, + 150033.061800003051758, + 0.0, + 22842.757500000298023, + 150063.555600002408028, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 22826.411499999463558, + 150063.357500001788139, + 0.0 + ], + [ + 22832.272399999201298, + 150060.221000000834465, + 0.0 + ], + [ + 22832.609200000762939, + 150060.850400000810623, + 0.0 + ], + [ + 22833.580099999904633, + 150062.664800003170967, + 0.0 + ], + [ + 22840.825800001621246, + 150058.787200000137091, + 0.0 + ], + [ + 22842.757500000298023, + 150057.753499999642372, + 0.0 + ], + [ + 22841.060200002044439, + 150054.581800002604723, + 0.0 + ], + [ + 22829.543500002473593, + 150033.061800003051758, + 0.0 + ], + [ + 22814.135000001639128, + 150041.30800000205636, + 0.0 + ], + [ + 22815.89299999922514, + 150044.593100000172853, + 0.0 + ], + [ + 22826.041400000452995, + 150063.555600002408028, + 0.0 + ], + [ + 22826.411499999463558, + 150063.357500001788139, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2565115, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23690149798", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 666.59224832899997, + "SHAPE_Area": 1754.8776121799999, + "AREA_SQFT": 18889.3 + }, + "bbox": [ + 23608.65260000154376, + 149786.040000002831221, + 0.0, + 23716.468699999153614, + 149934.448400001972914, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23677.989900000393391, + 149786.040000002831221, + 0.0 + ], + [ + 23608.65260000154376, + 149905.035600002855062, + 0.0 + ], + [ + 23659.130400002002716, + 149934.448400001972914, + 0.0 + ], + [ + 23709.182300001382828, + 149848.55010000243783, + 0.0 + ], + [ + 23702.254200000315905, + 149844.513300001621246, + 0.0 + ], + [ + 23653.967100001871586, + 149927.382800001651049, + 0.0 + ], + [ + 23613.26410000026226, + 149903.665699999779463, + 0.0 + ], + [ + 23676.700699999928474, + 149794.797200001776218, + 0.0 + ], + [ + 23712.332600001245737, + 149815.559600003063679, + 0.0 + ], + [ + 23716.468699999153614, + 149808.461100000888109, + 0.0 + ], + [ + 23677.989900000393391, + 149786.040000002831221, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2565175, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23677149868", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 208.79166312800001, + "SHAPE_Area": 756.18697303700003, + "AREA_SQFT": 8139.5 + }, + "bbox": [ + 23649.58049999922514, + 149824.107200000435114, + 0.0, + 23704.862700000405312, + 149911.536400001496077, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23698.09180000051856, + 149824.107200000435114, + 0.0 + ], + [ + 23649.58049999922514, + 149907.602500002831221, + 0.0 + ], + [ + 23656.351600002497435, + 149911.536400001496077, + 0.0 + ], + [ + 23704.862700000405312, + 149828.041200000792742, + 0.0 + ], + [ + 23698.09180000051856, + 149824.107200000435114, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2565183, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23665149861", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 207.67019654800001, + "SHAPE_Area": 732.90812511800004, + "AREA_SQFT": 7888.93 + }, + "bbox": [ + 23637.763700000941753, + 149817.533100001513958, + 0.0, + 23692.70160000026226, + 149904.5471, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23686.116100002080202, + 149817.533100001513958, + 0.0 + ], + [ + 23637.763700000941753, + 149900.719300001859665, + 0.0 + ], + [ + 23644.349199999123812, + 149904.5471, + 0.0 + ], + [ + 23692.70160000026226, + 149821.361000001430511, + 0.0 + ], + [ + 23686.116100002080202, + 149817.533100001513958, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2565190, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23742149880", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 122.36290404099999, + "SHAPE_Area": 646.52232617100003, + "AREA_SQFT": 6959.08 + }, + "bbox": [ + 23724.983899999409914, + 149858.991399999707937, + 0.0, + 23760.777000002563, + 149899.87780000269413, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23738.19200000166893, + 149899.87780000269413, + 0.0 + ], + [ + 23760.777000002563, + 149866.160600002855062, + 0.0 + ], + [ + 23756.139400001615286, + 149863.332299999892712, + 0.0 + ], + [ + 23749.021099999547005, + 149858.991399999707937, + 0.0 + ], + [ + 23739.843000002205372, + 149873.353400003165007, + 0.0 + ], + [ + 23738.434500001370907, + 149872.516200002282858, + 0.0 + ], + [ + 23736.961800001561642, + 149875.730600003153086, + 0.0 + ], + [ + 23733.695100001990795, + 149873.934000000357628, + 0.0 + ], + [ + 23731.586500000208616, + 149877.310600001364946, + 0.0 + ], + [ + 23726.082499999552965, + 149886.124200001358986, + 0.0 + ], + [ + 23728.469000000506639, + 149888.12780000269413, + 0.0 + ], + [ + 23726.007200002670288, + 149892.045700002461672, + 0.0 + ], + [ + 23724.983899999409914, + 149893.5912000015378, + 0.0 + ], + [ + 23727.842800002545118, + 149895.429100003093481, + 0.0 + ], + [ + 23727.374400001019239, + 149896.157600000500679, + 0.0 + ], + [ + 23733.05970000103116, + 149899.811800003051758, + 0.0 + ], + [ + 23734.530200000852346, + 149897.524100001901388, + 0.0 + ], + [ + 23738.19200000166893, + 149899.87780000269413, + 0.0 + ] + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "OBJECTID": 2565193, + "BLDGHEIGHT": 0, + "NUMSTORIES": 0, + "FEATURECOD": null, + "LASTUPDATE": "2017\/08\/15", + "LASTEDITOR": "VGIN", + "RuleID": 0, + "BUILDINGCL": 0, + "DATASOURCE": "Jurisdiction", + "EDITCOMMEN": null, + "SOURCEFEAT": "23654149854", + "RuleID_1": 1, + "SFIDdupes": 0, + "FIPS": "51680", + "MUNICIPALI": "Lynchburg City", + "SHAPE_Leng": 207.14953045, + "SHAPE_Area": 744.31082211199998, + "AREA_SQFT": 8011.66 + }, + "bbox": [ + 23626.108500000089407, + 149810.810000002384186, + 0.0, + 23681.160199999809265, + 149897.444099999964, + 0.0 + ], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 23674.452899999916553, + 149810.810000002384186, + 0.0 + ], + [ + 23626.108500000089407, + 149893.523800000548363, + 0.0 + ], + [ + 23632.815799999982119, + 149897.444099999964, + 0.0 + ], + [ + 23681.160199999809265, + 149814.73030000180006, + 0.0 + ], + [ + 23674.452899999916553, + 149810.810000002384186, + 0.0 + ] + ] + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/pyrasterframes/src/test/resources/luray-labels.geojson b/pyrasterframes/src/test/resources/luray-labels.geojson new file mode 100644 index 000000000..bdcbcb504 --- /dev/null +++ b/pyrasterframes/src/test/resources/luray-labels.geojson @@ -0,0 +1,18 @@ +{ +"type": "FeatureCollection", +"name": "L8-Labels-Elkton-VA", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -78.074908035755897, 38.325593965393153 ], [ -78.078776658402262, 38.327823897217776 ], [ -78.092819443861529, 38.32571146899194 ], [ -78.100161396610659, 38.313995346890302 ], [ -78.089376916671426, 38.31282539893752 ], [ -78.077634077459933, 38.317374423878938 ], [ -78.073898621221971, 38.322204512883999 ], [ -78.074908035755897, 38.325593965393153 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -78.369170650882523, 38.58065284389172 ], [ -78.383640370680567, 38.54472654334991 ], [ -78.327374251542963, 38.538843940894999 ], [ -78.314259254758397, 38.575748016690518 ], [ -78.369170650882523, 38.58065284389172 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.565452768282938, 38.692698239200176 ], [ -77.54853756075201, 38.687441203624303 ], [ -77.547985098699257, 38.68151785733005 ], [ -77.564314874957617, 38.685855270536351 ], [ -77.565452768282938, 38.692698239200176 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -78.625973620483947, 38.384086145975928 ], [ -78.619781683426993, 38.388373376437947 ], [ -78.613688820548219, 38.386918574305838 ], [ -78.609339103717076, 38.395958714056761 ], [ -78.600276924123435, 38.395057914051847 ], [ -78.598004559194038, 38.384358026518214 ], [ -78.600712105415838, 38.372726630839622 ], [ -78.61155677616398, 38.374617746567012 ], [ -78.625973620483947, 38.384086145975928 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 3 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.499872299697259, 38.265959576364672 ], [ -77.490962536589549, 38.262548509310811 ], [ -77.494072526256261, 38.251792574227593 ], [ -77.508081452628034, 38.250717051596098 ], [ -77.512193148300739, 38.258543877561848 ], [ -77.499872299697259, 38.265959576364672 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.896560416594369, 38.384489202082833 ], [ -77.885446228394855, 38.379901618618995 ], [ -77.886558613990559, 38.386926446793112 ], [ -77.892167110596205, 38.388029313414151 ], [ -77.896560416594369, 38.384489202082833 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -78.104587721849782, 38.230347976602772 ], [ -78.096623830061731, 38.232933469701152 ], [ -78.093733463904584, 38.227058631140601 ], [ -78.100422718389751, 38.223567355393556 ], [ -78.104587721849782, 38.230347976602772 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -78.09500918449011, 38.235516869468242 ], [ -78.084617397721388, 38.240823731215684 ], [ -78.079911154013928, 38.23752678606543 ], [ -78.084755595273549, 38.232321061869477 ], [ -78.09500918449011, 38.235516869468242 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 3 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.505050042952476, 38.306327713885693 ], [ -77.5048207527187, 38.300040419530752 ], [ -77.511898783174473, 38.289086070422542 ], [ -77.522735680615511, 38.292059924398963 ], [ -77.514377553898413, 38.300185554775908 ], [ -77.513706480506912, 38.310493467061114 ], [ -77.505050042952476, 38.306327713885693 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.89809802814473, 38.532542129891446 ], [ -77.901444154311747, 38.525317699576725 ], [ -77.893247775998773, 38.518264149754522 ], [ -77.897703956491853, 38.513453996400983 ], [ -77.901049016994193, 38.520378827806503 ], [ -77.916274443433977, 38.528730212529119 ], [ -77.909492522815626, 38.535864361205661 ], [ -77.89809802814473, 38.532542129891446 ] ] ] } }, +{ "type": "Feature", "properties": { "id": 3 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -77.516032435253635, 38.794413501916893 ], [ -77.523118384311459, 38.785034143507744 ], [ -77.505102142982878, 38.775534328871899 ], [ -77.504411687954828, 38.760891842932352 ], [ -77.483604951868131, 38.747060201442238 ], [ -77.463488423830327, 38.747867575574077 ], [ -77.449558893048575, 38.757506997981018 ], [ -77.516032435253635, 38.794413501916893 ] ] ] } } +] +} diff --git a/rf-notebook/README.md b/rf-notebook/README.md new file mode 100644 index 000000000..5295b9cc5 --- /dev/null +++ b/rf-notebook/README.md @@ -0,0 +1,32 @@ +# RasterFrames Jupyter Notebook Docker Image + +RasterFrames provides a Docker image with a Jupyter Notebook pre-configured with RasterFrames in the PySpark kernel. + +## Launching RasterFrames Notebook from Source + +First, have Docker installed. + +Then, from the source root directory, run + +```bash +sbt rf-notebook/startRFNotebook +``` + +This will build the project, create a Docker image, and start an instance. + +To stop it: + +```bash +sbt rf-notebook/stopRFNotebook +``` + +The above commands are just conveniences. The real work is done by `docker-compose` via the [`docker-compose.yml`](src/main/docker/docker-compose.yml) file in `rf-notebook/src/main/docker/docker-compose.yml`. By default it exposes and binds to ports `8888` and `4040-4044`. + + +## Docker Image Build + +If you just want to build the image: + +```bash +sbt rf-notebook/publishLocal +``` diff --git a/rf-notebook/build.sbt b/rf-notebook/build.sbt new file mode 100644 index 000000000..cac2f88ce --- /dev/null +++ b/rf-notebook/build.sbt @@ -0,0 +1,68 @@ +import scala.sys.process.Process +import PythonBuildPlugin.autoImport.pyWhl + +lazy val includeNotebooks = settingKey[Boolean]("Whether to build documentation into notebooks and include them") +includeNotebooks := true + +Docker / packageName := "rasterframes-notebook" + +Docker / version := version.value + +Docker / maintainer := organization.value + +Docker / sourceDirectory := baseDirectory.value / "src"/ "main" / "docker" + +Docker / target := target.value / "docker" + +dockerUpdateLatest := true + +Docker / mappings := Def.sequential( + LocalProject("pyrasterframes") / pyWhl, + Def.task { + val dockerSrc = (Docker / sourceDirectory).value + val dockerAssets = (dockerSrc ** "*") pair Path.relativeTo(dockerSrc) + + val py = (LocalProject("pyrasterframes") / pyWhl).value + + Def.taskDyn { + val withNB = includeNotebooks.value + if (withNB) + (LocalProject("pyrasterframes") / pySetup).toTask(" notebooks") + else Def.task(0) + }.value + + val nbFiles = ((LocalProject("pyrasterframes") / Python / doc / target).value ** "*.ipynb").get() + + val examples = nbFiles.map(f => (f, "examples/" + f.getName)) + dockerAssets ++ Seq(py -> py.getName) ++ examples + } +).value + +// This bypasses the standard DockerPlugin DSL-based Dockerfile construction +// and just copies the separate, external one. +Docker / dockerGenerateConfig := (Docker / sourceDirectory).value / "Dockerfile" + +// Save a bit of typing... +publishLocal := (Docker / publishLocal).value + +// -----== Conveniences ==----- + +lazy val startRFNotebook = Def.taskKey[Unit]("Build and run a RasterFrames Notebook Docker container") +lazy val quickStartRFNotebook = Def.taskKey[Unit]("Run RasterFrames Notebook Docker container without building it first") +lazy val stopRFNotebook = Def.taskKey[Unit]("Stop a running RasterFrames Notebook instance") + +startRFNotebook := { + val _ = (Docker / publishLocal).value + val staging = (Docker / stagingDirectory).value + Process("docker-compose up --force-recreate -d", staging).! +} + +quickStartRFNotebook := { + val staging = (Docker / stagingDirectory).value + Process("docker-compose up --force-recreate -d", staging).! +} + +stopRFNotebook := { + val staging = (Docker / stagingDirectory).value + Process("docker-compose down", staging).! +} diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile new file mode 100644 index 000000000..ad430a9f7 --- /dev/null +++ b/rf-notebook/src/main/docker/Dockerfile @@ -0,0 +1,33 @@ +FROM s22s/pyspark-notebook:spark-2.3.3-hadoop-2.7 + +MAINTAINER Astraea, Inc. + +ENV RF_LIB_LOC=/usr/local/rasterframes \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib" + +USER root + +RUN mkdir $RF_LIB_LOC + +EXPOSE 4040 4041 4042 4043 4044 + +# Sphinx (for Notebook->html) +RUN conda install --quiet --yes \ + anaconda sphinx nbsphinx shapely numpy folium geopandas geojsonio + +# Cleanup pip residuals +RUN rm -rf /home/$NB_USER/.local && \ + fix-permissions /home/$NB_USER + +# Note: The above step takes an insanely long time in the CONDA_DIR, so commenting it out until we have perm issues. +# fix-permissions $CONDA_DIR + +COPY *.whl $RF_LIB_LOC +RUN ls -1 $RF_LIB_LOC/*.whl | xargs pip install +COPY jupyter_notebook_config.py $HOME/.jupyter +COPY examples $HOME/examples +RUN chmod -R +w $HOME/examples + +RUN chown -R $NB_UID:$NB_GID $HOME + +USER $NB_UID \ No newline at end of file diff --git a/rf-notebook/src/main/docker/docker-compose.yml b/rf-notebook/src/main/docker/docker-compose.yml new file mode 100644 index 000000000..79d6c8dcf --- /dev/null +++ b/rf-notebook/src/main/docker/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + rasterframes-notebook: + image: rasterframes-notebook + ports: + # jupyter notebook port + - "8888:8888" + # spark UI ports + - "44040:4040" + - "44041:4041" + - "44042:4042" + - "44043:4043" + - "44044:4044" +# To save locally at './work' from the container: + volumes: + - ../../../src/main/notebooks:/home/jovyan/work + - ~/.rf_cache:/home/jovyan/.rf_cache \ No newline at end of file diff --git a/rf-notebook/src/main/docker/jupyter_notebook_config.py b/rf-notebook/src/main/docker/jupyter_notebook_config.py new file mode 100644 index 000000000..ceb183c50 --- /dev/null +++ b/rf-notebook/src/main/docker/jupyter_notebook_config.py @@ -0,0 +1,766 @@ +# Configuration file for PyRasterFrames-enabled jupyter-notebook. + +#------------------------------------------------------------------------------ +# Application(SingletonConfigurable) configuration +#------------------------------------------------------------------------------ + +## This is an application. + +## The date format used by logging formatters for %(asctime)s +#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S' + +## The Logging format template +#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s' + +## Set the log level by value or name. +#c.Application.log_level = 30 + +#------------------------------------------------------------------------------ +# JupyterApp(Application) configuration +#------------------------------------------------------------------------------ + +## Base class for Jupyter applications + +## Answer yes to any prompts. +#c.JupyterApp.answer_yes = False + +## Full path of a config file. +#c.JupyterApp.config_file = '' + +## Specify a config file to load. +#c.JupyterApp.config_file_name = '' + +## Generate default config file. +#c.JupyterApp.generate_config = False + +#------------------------------------------------------------------------------ +# NotebookApp(JupyterApp) configuration +#------------------------------------------------------------------------------ + +## Set the Access-Control-Allow-Credentials: true header +#c.NotebookApp.allow_credentials = False + +## Set the Access-Control-Allow-Origin header +# +# Use '*' to allow any origin to access your server. +# +# Takes precedence over allow_origin_pat. +#c.NotebookApp.allow_origin = '' + +## Use a regular expression for the Access-Control-Allow-Origin header +# +# Requests from an origin matching the expression will get replies with: +# +# Access-Control-Allow-Origin: origin +# +# where `origin` is the origin of the request. +# +# Ignored if allow_origin is set. +#c.NotebookApp.allow_origin_pat = '' + +## Allow password to be changed at login for the notebook server. +# +# While loggin in with a token, the notebook server UI will give the opportunity +# to the user to enter a new password at the same time that will replace the +# token login mechanism. +# +# This can be set to false to prevent changing password from the UI/API. +#c.NotebookApp.allow_password_change = True + +## Allow requests where the Host header doesn't point to a local server +# +# By default, requests get a 403 forbidden response if the 'Host' header shows +# that the browser thinks it's on a non-local domain. Setting this option to +# True disables this check. +# +# This protects against 'DNS rebinding' attacks, where a remote web server +# serves you a page and then changes its DNS to send later requests to a local +# IP, bypassing same-origin checks. +# +# Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, along +# with hostnames configured in local_hostnames. +#c.NotebookApp.allow_remote_access = False + +## Whether to allow the user to run the notebook as root. +#c.NotebookApp.allow_root = False + +## DEPRECATED use base_url +#c.NotebookApp.base_project_url = '/' + +## The base URL for the notebook server. +# +# Leading and trailing slashes can be omitted, and will automatically be added. +#c.NotebookApp.base_url = '/' + +## Specify what command to use to invoke a web browser when opening the notebook. +# If not specified, the default browser will be determined by the `webbrowser` +# standard library module, which allows setting of the BROWSER environment +# variable to override it. +#c.NotebookApp.browser = '' + +## The full path to an SSL/TLS certificate file. +#c.NotebookApp.certfile = '' + +## The full path to a certificate authority certificate for SSL/TLS client +# authentication. +#c.NotebookApp.client_ca = '' + +## The config manager class to use +#c.NotebookApp.config_manager_class = 'notebook.services.config.manager.ConfigManager' + +## The notebook manager class to use. +#c.NotebookApp.contents_manager_class = 'notebook.services.contents.largefilemanager.LargeFileManager' + +## Extra keyword arguments to pass to `set_secure_cookie`. See tornado's +# set_secure_cookie docs for details. +#c.NotebookApp.cookie_options = {} + +## The random bytes used to secure cookies. By default this is a new random +# number every time you start the Notebook. Set it to a value in a config file +# to enable logins to persist across server sessions. +# +# Note: Cookie secrets should be kept private, do not share config files with +# cookie_secret stored in plaintext (you can read the value from a file). +#c.NotebookApp.cookie_secret = b'' + +## The file where the cookie secret is stored. +#c.NotebookApp.cookie_secret_file = '' + +## Override URL shown to users. +# +# Replace actual URL, including protocol, address, port and base URL, with the +# given value when displaying URL to the users. Do not change the actual +# connection URL. If authentication token is enabled, the token is added to the +# custom URL automatically. +# +# This option is intended to be used when the URL to display to the user cannot +# be determined reliably by the Jupyter notebook server (proxified or +# containerized setups for example). +#c.NotebookApp.custom_display_url = '' + +## The default URL to redirect to from `/` +#c.NotebookApp.default_url = '/tree' + +## Disable cross-site-request-forgery protection +# +# Jupyter notebook 4.3.1 introduces protection from cross-site request +# forgeries, requiring API requests to either: +# +# - originate from pages served by this server (validated with XSRF cookie and +# token), or - authenticate with a token +# +# Some anonymous compute resources still desire the ability to run code, +# completely without authentication. These services can disable all +# authentication and security checks, with the full knowledge of what that +# implies. +#c.NotebookApp.disable_check_xsrf = False + +## Whether to enable MathJax for typesetting math/TeX +# +# MathJax is the javascript library Jupyter uses to render math/LaTeX. It is +# very large, so you may want to disable it if you have a slow internet +# connection, or for offline use of the notebook. +# +# When disabled, equations etc. will appear as their untransformed TeX source. +#c.NotebookApp.enable_mathjax = True + +## extra paths to look for Javascript notebook extensions +#c.NotebookApp.extra_nbextensions_path = [] + +## handlers that should be loaded at higher priority than the default services +#c.NotebookApp.extra_services = [] + +## Extra paths to search for serving static files. +# +# This allows adding javascript/css to be available from the notebook server +# machine, or overriding individual files in the IPython +#c.NotebookApp.extra_static_paths = [] + +## Extra paths to search for serving jinja templates. +# +# Can be used to override templates from notebook.templates. +#c.NotebookApp.extra_template_paths = [] + +## +#c.NotebookApp.file_to_run = '' + +## Extra keyword arguments to pass to `get_secure_cookie`. See tornado's +# get_secure_cookie docs for details. +#c.NotebookApp.get_secure_cookie_kwargs = {} + +## Deprecated: Use minified JS file or not, mainly use during dev to avoid JS +# recompilation +#c.NotebookApp.ignore_minified_js = False + +## (bytes/sec) Maximum rate at which stream output can be sent on iopub before +# they are limited. +#c.NotebookApp.iopub_data_rate_limit = 1000000 + +## (msgs/sec) Maximum rate at which messages can be sent on iopub before they are +# limited. +#c.NotebookApp.iopub_msg_rate_limit = 1000 + +## The IP address the notebook server will listen on. +# c.NotebookApp.ip = 'localhost' + +## Supply extra arguments that will be passed to Jinja environment. +#c.NotebookApp.jinja_environment_options = {} + +## Extra variables to supply to jinja templates when rendering. +#c.NotebookApp.jinja_template_vars = {} + +## The kernel manager class to use. +#c.NotebookApp.kernel_manager_class = 'notebook.services.kernels.kernelmanager.MappingKernelManager' + +## The kernel spec manager class to use. Should be a subclass of +# `jupyter_client.kernelspec.KernelSpecManager`. +# +# The Api of KernelSpecManager is provisional and might change without warning +# between this version of Jupyter and the next stable one. +#c.NotebookApp.kernel_spec_manager_class = 'jupyter_client.kernelspec.KernelSpecManager' + +## The full path to a private key file for usage with SSL/TLS. +#c.NotebookApp.keyfile = '' + +## Hostnames to allow as local when allow_remote_access is False. +# +# Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted as +# local as well. +#c.NotebookApp.local_hostnames = ['localhost'] + +## The login handler class to use. +#c.NotebookApp.login_handler_class = 'notebook.auth.login.LoginHandler' + +## The logout handler class to use. +#c.NotebookApp.logout_handler_class = 'notebook.auth.logout.LogoutHandler' + +## The MathJax.js configuration file that is to be used. +#c.NotebookApp.mathjax_config = 'TeX-AMS-MML_HTMLorMML-full,Safe' + +## A custom url for MathJax.js. Should be in the form of a case-sensitive url to +# MathJax, for example: /static/components/MathJax/MathJax.js +#c.NotebookApp.mathjax_url = '' + +## Sets the maximum allowed size of the client request body, specified in the +# Content-Length request header field. If the size in a request exceeds the +# configured value, a malformed HTTP message is returned to the client. +# +# Note: max_body_size is applied even in streaming mode. +#c.NotebookApp.max_body_size = 536870912 + +## Gets or sets the maximum amount of memory, in bytes, that is allocated for +# use by the buffer manager. +#c.NotebookApp.max_buffer_size = 536870912 + +## Dict of Python modules to load as notebook server extensions.Entry values can +# be used to enable and disable the loading ofthe extensions. The extensions +# will be loaded in alphabetical order. +#c.NotebookApp.nbserver_extensions = {} + +## The directory to use for notebooks and kernels. +#c.NotebookApp.notebook_dir = '' + +## Whether to open in a browser after starting. The specific browser used is +# platform dependent and determined by the python standard library `webbrowser` +# module, unless it is overridden using the --browser (NotebookApp.browser) +# configuration option. +#c.NotebookApp.open_browser = True + +## Hashed password to use for web authentication. +# +# To generate, type in a python/IPython shell: +# +# from notebook.auth import passwd; passwd() +# +# The string should be of the form type:salt:hashed-password. +#c.NotebookApp.password = '' + +## Forces users to use a password for the Notebook server. This is useful in a +# multi user environment, for instance when everybody in the LAN can access each +# other's machine through ssh. +# +# In such a case, server the notebook server on localhost is not secure since +# any user can connect to the notebook server via ssh. +#c.NotebookApp.password_required = False + +## The port the notebook server will listen on. +#c.NotebookApp.port = 8888 + +## The number of additional ports to try if the specified port is not available. +#c.NotebookApp.port_retries = 50 + +## DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. +#c.NotebookApp.pylab = 'disabled' + +## If True, display a button in the dashboard to quit (shutdown the notebook +# server). +#c.NotebookApp.quit_button = True + +## (sec) Time window used to check the message and data rate limits. +#c.NotebookApp.rate_limit_window = 3 + +## Reraise exceptions encountered loading server extensions? +#c.NotebookApp.reraise_server_extension_failures = False + +## DEPRECATED use the nbserver_extensions dict instead +#c.NotebookApp.server_extensions = [] + +## The session manager class to use. +#c.NotebookApp.session_manager_class = 'notebook.services.sessions.sessionmanager.SessionManager' + +## Shut down the server after N seconds with no kernels or terminals running and +# no activity. This can be used together with culling idle kernels +# (MappingKernelManager.cull_idle_timeout) to shutdown the notebook server when +# it's not in use. This is not precisely timed: it may shut down up to a minute +# later. 0 (the default) disables this automatic shutdown. +#c.NotebookApp.shutdown_no_activity_timeout = 0 + +## Supply SSL options for the tornado HTTPServer. See the tornado docs for +# details. +#c.NotebookApp.ssl_options = {} + +## Supply overrides for terminado. Currently only supports "shell_command". +#c.NotebookApp.terminado_settings = {} + +## Set to False to disable terminals. +# +# This does *not* make the notebook server more secure by itself. Anything the +# user can in a terminal, they can also do in a notebook. +# +# Terminals may also be automatically disabled if the terminado package is not +# available. +#c.NotebookApp.terminals_enabled = True + +## Token used for authenticating first-time connections to the server. +# +# When no password is enabled, the default is to generate a new, random token. +# +# Setting to an empty string disables authentication altogether, which is NOT +# RECOMMENDED. +#c.NotebookApp.token = '' +c.NotebookApp.token = '' + +## Supply overrides for the tornado.web.Application that the Jupyter notebook +# uses. +#c.NotebookApp.tornado_settings = {} + +## Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded- +# For headerssent by the upstream reverse proxy. Necessary if the proxy handles +# SSL +#c.NotebookApp.trust_xheaders = False + +## DEPRECATED, use tornado_settings +#c.NotebookApp.webapp_settings = {} + +## Specify Where to open the notebook on startup. This is the `new` argument +# passed to the standard library method `webbrowser.open`. The behaviour is not +# guaranteed, but depends on browser support. Valid values are: +# +# - 2 opens a new tab, +# - 1 opens a new window, +# - 0 opens in an existing window. +# +# See the `webbrowser.open` documentation for details. +#c.NotebookApp.webbrowser_open_new = 2 + +## Set the tornado compression options for websocket connections. +# +# This value will be returned from +# :meth:`WebSocketHandler.get_compression_options`. None (default) will disable +# compression. A dict (even an empty one) will enable compression. +# +# See the tornado docs for WebSocketHandler.get_compression_options for details. +#c.NotebookApp.websocket_compression_options = None + +## The base URL for websockets, if it differs from the HTTP server (hint: it +# almost certainly doesn't). +# +# Should be in the form of an HTTP origin: ws[s]://hostname[:port] +#c.NotebookApp.websocket_url = '' + +#------------------------------------------------------------------------------ +# ConnectionFileMixin(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## Mixin for configurable classes that work with connection files + +## JSON file in which to store connection info [default: kernel-.json] +# +# This file will contain the IP, ports, and authentication key needed to connect +# clients to this kernel. By default, this file will be created in the security +# dir of the current profile, but can be specified by absolute path. +#c.ConnectionFileMixin.connection_file = '' + +## set the control (ROUTER) port [default: random] +#c.ConnectionFileMixin.control_port = 0 + +## set the heartbeat port [default: random] +#c.ConnectionFileMixin.hb_port = 0 + +## set the iopub (PUB) port [default: random] +#c.ConnectionFileMixin.iopub_port = 0 + +## Set the kernel's IP address [default localhost]. If the IP address is +# something other than localhost, then Consoles on other machines will be able +# to connect to the Kernel, so be careful! +#c.ConnectionFileMixin.ip = '' + +## set the shell (ROUTER) port [default: random] +#c.ConnectionFileMixin.shell_port = 0 + +## set the stdin (ROUTER) port [default: random] +#c.ConnectionFileMixin.stdin_port = 0 + +## +#c.ConnectionFileMixin.transport = 'tcp' + +#------------------------------------------------------------------------------ +# KernelManager(ConnectionFileMixin) configuration +#------------------------------------------------------------------------------ + +## Manages a single kernel in a subprocess on this host. +# +# This version starts kernels with Popen. + +## Should we autorestart the kernel if it dies. +#c.KernelManager.autorestart = True + +## DEPRECATED: Use kernel_name instead. +# +# The Popen Command to launch the kernel. Override this if you have a custom +# kernel. If kernel_cmd is specified in a configuration file, Jupyter does not +# pass any arguments to the kernel, because it cannot make any assumptions about +# the arguments that the kernel understands. In particular, this means that the +# kernel does not receive the option --debug if it given on the Jupyter command +# line. +#c.KernelManager.kernel_cmd = [] + +## Time to wait for a kernel to terminate before killing it, in seconds. +#c.KernelManager.shutdown_wait_time = 5.0 + +#------------------------------------------------------------------------------ +# Session(Configurable) configuration +#------------------------------------------------------------------------------ + +## Object for handling serialization and sending of messages. +# +# The Session object handles building messages and sending them with ZMQ sockets +# or ZMQStream objects. Objects can communicate with each other over the +# network via Session objects, and only need to work with the dict-based IPython +# message spec. The Session will handle serialization/deserialization, security, +# and metadata. +# +# Sessions support configurable serialization via packer/unpacker traits, and +# signing with HMAC digests via the key/keyfile traits. +# +# Parameters ---------- +# +# debug : bool +# whether to trigger extra debugging statements +# packer/unpacker : str : 'json', 'pickle' or import_string +# importstrings for methods to serialize message parts. If just +# 'json' or 'pickle', predefined JSON and pickle packers will be used. +# Otherwise, the entire importstring must be used. +# +# The functions must accept at least valid JSON input, and output *bytes*. +# +# For example, to use msgpack: +# packer = 'msgpack.packb', unpacker='msgpack.unpackb' +# pack/unpack : callables +# You can also set the pack/unpack callables for serialization directly. +# session : bytes +# the ID of this Session object. The default is to generate a new UUID. +# username : unicode +# username added to message headers. The default is to ask the OS. +# key : bytes +# The key used to initialize an HMAC signature. If unset, messages +# will not be signed or checked. +# keyfile : filepath +# The file containing a key. If this is set, `key` will be initialized +# to the contents of the file. + +## Threshold (in bytes) beyond which an object's buffer should be extracted to +# avoid pickling. +#c.Session.buffer_threshold = 1024 + +## Whether to check PID to protect against calls after fork. +# +# This check can be disabled if fork-safety is handled elsewhere. +#c.Session.check_pid = True + +## Threshold (in bytes) beyond which a buffer should be sent without copying. +#c.Session.copy_threshold = 65536 + +## Debug output in the Session +#c.Session.debug = False + +## The maximum number of digests to remember. +# +# The digest history will be culled when it exceeds this value. +#c.Session.digest_history_size = 65536 + +## The maximum number of items for a container to be introspected for custom +# serialization. Containers larger than this are pickled outright. +#c.Session.item_threshold = 64 + +## execution key, for signing messages. +#c.Session.key = b'' + +## path to file containing execution key. +#c.Session.keyfile = '' + +## Metadata dictionary, which serves as the default top-level metadata dict for +# each message. +#c.Session.metadata = {} + +## The name of the packer for serializing messages. Should be one of 'json', +# 'pickle', or an import name for a custom callable serializer. +#c.Session.packer = 'json' + +## The UUID identifying this session. +#c.Session.session = '' + +## The digest scheme used to construct the message signatures. Must have the form +# 'hmac-HASH'. +#c.Session.signature_scheme = 'hmac-sha256' + +## The name of the unpacker for unserializing messages. Only used with custom +# functions for `packer`. +#c.Session.unpacker = 'json' + +## Username for the Session. Default is your system username. +#c.Session.username = 'username' + +#------------------------------------------------------------------------------ +# MultiKernelManager(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## A class for managing multiple kernels. + +## The name of the default kernel to start +#c.MultiKernelManager.default_kernel_name = 'python3' + +## The kernel manager class. This is configurable to allow subclassing of the +# KernelManager for customized behavior. +#c.MultiKernelManager.kernel_manager_class = 'jupyter_client.ioloop.IOLoopKernelManager' + +#------------------------------------------------------------------------------ +# MappingKernelManager(MultiKernelManager) configuration +#------------------------------------------------------------------------------ + +## A KernelManager that handles notebook mapping and HTTP error handling + +## Whether messages from kernels whose frontends have disconnected should be +# buffered in-memory. +# +# When True (default), messages are buffered and replayed on reconnect, avoiding +# lost messages due to interrupted connectivity. +# +# Disable if long-running kernels will produce too much output while no +# frontends are connected. +#c.MappingKernelManager.buffer_offline_messages = True + +## Whether to consider culling kernels which are busy. Only effective if +# cull_idle_timeout > 0. +#c.MappingKernelManager.cull_busy = False + +## Whether to consider culling kernels which have one or more connections. Only +# effective if cull_idle_timeout > 0. +#c.MappingKernelManager.cull_connected = False + +## Timeout (in seconds) after which a kernel is considered idle and ready to be +# culled. Values of 0 or lower disable culling. Very short timeouts may result +# in kernels being culled for users with poor network connections. +#c.MappingKernelManager.cull_idle_timeout = 0 + +## The interval (in seconds) on which to check for idle kernels exceeding the +# cull timeout value. +#c.MappingKernelManager.cull_interval = 300 + +## Timeout for giving up on a kernel (in seconds). +# +# On starting and restarting kernels, we check whether the kernel is running and +# responsive by sending kernel_info_requests. This sets the timeout in seconds +# for how long the kernel can take before being presumed dead. This affects the +# MappingKernelManager (which handles kernel restarts) and the +# ZMQChannelsHandler (which handles the startup). +#c.MappingKernelManager.kernel_info_timeout = 60 + +## +#c.MappingKernelManager.root_dir = '' + +#------------------------------------------------------------------------------ +# ContentsManager(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## Base class for serving files and directories. +# +# This serves any text or binary file, as well as directories, with special +# handling for JSON notebook documents. +# +# Most APIs take a path argument, which is always an API-style unicode path, and +# always refers to a directory. +# +# - unicode, not url-escaped +# - '/'-separated +# - leading and trailing '/' will be stripped +# - if unspecified, path defaults to '', +# indicating the root path. + +## Allow access to hidden files +#c.ContentsManager.allow_hidden = False + +## +#c.ContentsManager.checkpoints = None + +## +#c.ContentsManager.checkpoints_class = 'notebook.services.contents.checkpoints.Checkpoints' + +## +#c.ContentsManager.checkpoints_kwargs = {} + +## handler class to use when serving raw file requests. +# +# Default is a fallback that talks to the ContentsManager API, which may be +# inefficient, especially for large files. +# +# Local files-based ContentsManagers can use a StaticFileHandler subclass, which +# will be much more efficient. +# +# Access to these files should be Authenticated. +#c.ContentsManager.files_handler_class = 'notebook.files.handlers.FilesHandler' + +## Extra parameters to pass to files_handler_class. +# +# For example, StaticFileHandlers generally expect a `path` argument specifying +# the root directory from which to serve files. +#c.ContentsManager.files_handler_params = {} + +## Glob patterns to hide in file and directory listings. +#c.ContentsManager.hide_globs = ['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~'] + +## Python callable or importstring thereof +# +# To be called on a contents model prior to save. +# +# This can be used to process the structure, such as removing notebook outputs +# or other side effects that should not be saved. +# +# It will be called as (all arguments passed by keyword):: +# +# hook(path=path, model=model, contents_manager=self) +# +# - model: the model to be saved. Includes file contents. +# Modifying this dict will affect the file that is stored. +# - path: the API path of the save destination +# - contents_manager: this ContentsManager instance +#c.ContentsManager.pre_save_hook = None + +## +#c.ContentsManager.root_dir = '/' + +## The base name used when creating untitled directories. +#c.ContentsManager.untitled_directory = 'Untitled Folder' + +## The base name used when creating untitled files. +#c.ContentsManager.untitled_file = 'untitled' + +## The base name used when creating untitled notebooks. +#c.ContentsManager.untitled_notebook = 'Untitled' + +#------------------------------------------------------------------------------ +# FileManagerMixin(Configurable) configuration +#------------------------------------------------------------------------------ + +## Mixin for ContentsAPI classes that interact with the filesystem. +# +# Provides facilities for reading, writing, and copying both notebooks and +# generic files. +# +# Shared by FileContentsManager and FileCheckpoints. +# +# Note ---- Classes using this mixin must provide the following attributes: +# +# root_dir : unicode +# A directory against against which API-style paths are to be resolved. +# +# log : logging.Logger + +## By default notebooks are saved on disk on a temporary file and then if +# succefully written, it replaces the old ones. This procedure, namely +# 'atomic_writing', causes some bugs on file system whitout operation order +# enforcement (like some networked fs). If set to False, the new notebook is +# written directly on the old one which could fail (eg: full filesystem or quota +# ) +#c.FileManagerMixin.use_atomic_writing = True + +#------------------------------------------------------------------------------ +# FileContentsManager(FileManagerMixin,ContentsManager) configuration +#------------------------------------------------------------------------------ + +## If True (default), deleting files will send them to the platform's +# trash/recycle bin, where they can be recovered. If False, deleting files +# really deletes them. +#c.FileContentsManager.delete_to_trash = True + +## Python callable or importstring thereof +# +# to be called on the path of a file just saved. +# +# This can be used to process the file on disk, such as converting the notebook +# to a script or HTML via nbconvert. +# +# It will be called as (all arguments passed by keyword):: +# +# hook(os_path=os_path, model=model, contents_manager=instance) +# +# - path: the filesystem path to the file just written - model: the model +# representing the file - contents_manager: this ContentsManager instance +#c.FileContentsManager.post_save_hook = None + +## +#c.FileContentsManager.root_dir = '' + +## DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0 +#c.FileContentsManager.save_script = False + +#------------------------------------------------------------------------------ +# NotebookNotary(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## A class for computing and verifying notebook signatures. + +## The hashing algorithm used to sign notebooks. +#c.NotebookNotary.algorithm = 'sha256' + +## The sqlite file in which to store notebook signatures. By default, this will +# be in your Jupyter data directory. You can set it to ':memory:' to disable +# sqlite writing to the filesystem. +#c.NotebookNotary.db_file = '' + +## The secret key with which notebooks are signed. +#c.NotebookNotary.secret = b'' + +## The file where the secret key is stored. +#c.NotebookNotary.secret_file = '' + +## A callable returning the storage backend for notebook signatures. The default +# uses an SQLite database. +#c.NotebookNotary.store_factory = traitlets.Undefined + +#------------------------------------------------------------------------------ +# KernelSpecManager(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## If there is no Python kernelspec registered and the IPython kernel is +# available, ensure it is added to the spec list. +#c.KernelSpecManager.ensure_native_kernel = True + +## The kernel spec class. This is configurable to allow subclassing of the +# KernelSpecManager for customized behavior. +#c.KernelSpecManager.kernel_spec_class = 'jupyter_client.kernelspec.KernelSpec' + +## Whitelist of allowed kernel names. +# +# By default, all installed kernels are allowed. +#c.KernelSpecManager.whitelist = set() diff --git a/rf-notebook/src/main/notebooks/Getting Started.ipynb b/rf-notebook/src/main/notebooks/Getting Started.ipynb new file mode 100644 index 000000000..e299b4b2f --- /dev/null +++ b/rf-notebook/src/main/notebooks/Getting Started.ipynb @@ -0,0 +1,398 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting Started with RasterFrames Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Spark Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", + "from pyrasterframes.rasterfunctions import (rf_local_add, rf_dimensions, rf_extent, rf_crs, rf_mk_crs,\n", + " st_geometry, st_reproject, rf_tile)\n", + "import pyspark.sql.functions as F" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "spark = pyrasterframes.get_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get a PySpark DataFrame from [open data](https://docs.opendata.aws/modis-pds/readme.html)\n", + "\n", + "Read a single \"granule\" or scene of MODIS surface reflectance data. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "uri = 'https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059' \\\n", + " '/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF'\n", + "df = spark.read.raster(uri)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n", + " |-- proj_raster_path: string (nullable = false)\n", + " |-- proj_raster: struct (nullable = true)\n", + " | |-- tile_context: struct (nullable = false)\n", + " | | |-- extent: struct (nullable = false)\n", + " | | | |-- xmin: double (nullable = false)\n", + " | | | |-- ymin: double (nullable = false)\n", + " | | | |-- xmax: double (nullable = false)\n", + " | | | |-- ymax: double (nullable = false)\n", + " | | |-- crs: struct (nullable = false)\n", + " | | | |-- crsProj4: string (nullable = false)\n", + " | |-- tile: tile (nullable = false)\n", + "\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do some work with the raster data; add 3 element-wise to the pixel/cell values and show some rows of the DataFrame." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "|rf_local_add(proj_raster, 3) |\n", + "+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "|[[[-7783653.637667, 993342.4642358534, -7665045.582235852, 1111950.519667], [+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ]], [int16ud32767, (256,256), [3408,3471,3110,2875,2798,2973,3255,3169,-2147483648,3217,...,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,2519,3036,-2147483648]]]|\n", + "|[[[-7665045.582235853, 993342.4642358534, -7546437.526804706, 1111950.519667], [+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ]], [int16ud32767, (256,256), [2337,2346,2581,2751,2575,2364,2223,2384,2618,2296,...,-2147483648,-2147483648,2688,2702,2967,3200,3257,3052,2914,2534]]] |\n", + "|[[[-7546437.526804707, 993342.4642358534, -7427829.471373559, 1111950.519667], [+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ]], [int16ud32767, (257,256), [2602,2728,2784,2781,2567,2539,2254,2327,2436,2888,...,2726,2695,2788,2898,3139,3121,2939,2778,2859,2728]]] |\n", + "|[[[-7427829.47137356, 993342.4642358534, -7309221.415942413, 1111950.519667], [+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ]], [int16ud32767, (256,256), [3058,3163,3036,3228,2877,3310,2885,2932,2931,2940,...,2299,2190,2180,2441,2563,2431,2347,2414,2525,2778]]] |\n", + "|[[[-7309221.415942414, 993342.4642358534, -7190613.360511266, 1111950.519667], [+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ]], [int16ud32767, (257,256), [3238,3355,3502,3055,3343,3334,-2147483648,-2147483648,-2147483648,-2147483648,...,2629,2841,2983,2839,3107,2762,2524,3175,3190,3181]]] |\n", + "+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "only showing top 5 rows\n", + "\n" + ] + } + ], + "source": [ + "df.select(rf_local_add(df.proj_raster, F.lit(3))).show(5, False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The extent struct tells us where in the [CRS](https://spatialreference.org/ref/sr-org/6842/) the tile data covers. The granule is split into arbitrary sized chunks. Each row is a different chunk. Let's see how many.\n", + "\n", + "Side note: you can configure the default size of these chunks, which are called Tiles, by passing a tuple of desired columns and rows as: `raster(uri, tile_dimensions=(96, 96))`. The default is `(256, 256)`" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "100" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.count()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What area does the DataFrame cover?" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m \n", + "+--------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "|proj_raster_path |footprint |\n", + "+--------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-70.85954815687087 8.933333332533772, -71.07986282542622 9.999999999104968, -69.99674110618135 9.999999999104968, -69.7797836135278 8.933333332533772, -70.85954815687087 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-69.77978361352781 8.933333332533772, -69.99674110618135 9.999999999104968, -68.91361938693649 9.999999999104968, -68.70001907018472 8.933333332533772, -69.77978361352781 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-68.70001907018474 8.933333332533772, -68.9136193869365 9.999999999104968, -67.8304976676916 9.999999999104968, -67.62025452684163 8.933333332533772, -68.70001907018474 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-67.62025452684165 8.933333332533772, -67.83049766769162 9.999999999104968, -66.74737594844675 9.999999999104968, -66.54048998349857 8.933333332533772, -67.62025452684165 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-66.54048998349859 8.933333332533772, -66.74737594844676 9.999999999104968, -65.66425422920187 9.999999999104968, -65.4607254401555 8.933333332533772, -66.54048998349859 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-65.4607254401555 8.933333332533772, -65.66425422920187 9.999999999104968, -64.58113250995702 9.999999999104968, -64.38096089681244 8.933333332533772, -65.4607254401555 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-64.38096089681244 8.933333332533772, -64.58113250995702 9.999999999104968, -63.498010790712144 9.999999999104968, -63.30119635346936 8.933333332533772, -64.38096089681244 8.933333332533772))|\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-63.30119635346937 8.933333332533772, -63.49801079071215 9.999999999104968, -62.41488907146726 9.999999999104968, -62.221431810126276 8.933333332533772, -63.30119635346937 8.933333332533772))|\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-62.22143181012629 8.933333332533772, -62.41488907146727 9.999999999104968, -61.33176735222239 9.999999999104968, -61.14166726678321 8.933333332533772, -62.22143181012629 8.933333332533772)) |\n", + "|https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF|POLYGON ((-61.14166726678322 8.933333332533772, -61.3317673522224 9.999999999104968, -60.92559670750556 9.999999999104968, -60.736755563029554 8.933333332533772, -61.14166726678322 8.933333332533772)) |\n", + "+--------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "only showing top 10 rows\n", + "\n" + ] + } + ], + "source": [ + "crs = df.agg(F.first(rf_crs(df.proj_raster)).crsProj4.alias('crs')).first()['crs']\n", + "print(crs)\n", + "coverage_area = df.select(\n", + " df.proj_raster_path,\n", + " st_reproject(\n", + " st_geometry(rf_extent(df.proj_raster)), \n", + " rf_mk_crs(crs), \n", + " rf_mk_crs('EPSG:4326')).alias('footprint')\n", + " )\n", + "coverage_area.show(10, False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So where in the world is that? We'll generate a little visualization with Leaflet in the notebook using Folium." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import geopandas\n", + "import folium" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "gdf = geopandas.GeoDataFrame(\n", + " coverage_area.select('footprint').toPandas(), \n", + " geometry='footprint', crs={'init':'EPSG:4326'}) " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
                          " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "folium.Map((5, -65), zoom_start=6) \\\n", + " .add_child(folium.GeoJson(gdf.__geo_interface__))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Look at a sample of the data. You may find it useful to double-click the tile image column to see larger or smaller rendering of the image." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
                          \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
                          proj_raster_pathextenttile
                          0https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF(-7783653.637667, 993342.4642358534, -7665045.582235852, 1111950.519667)
                          1https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF(-7665045.582235853, 993342.4642358534, -7546437.526804706, 1111950.519667)
                          2https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF(-7546437.526804707, 993342.4642358534, -7427829.471373559, 1111950.519667)
                          3https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF(-7427829.47137356, 993342.4642358534, -7309221.415942413, 1111950.519667)
                          4https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF(-7309221.415942414, 993342.4642358534, -7190613.360511266, 1111950.519667)
                          \n", + "
                          " + ], + "text/plain": [ + " proj_raster_path \\\n", + "0 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "1 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "2 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "3 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "4 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "\n", + " extent \\\n", + "0 (-7783653.637667, 993342.4642358534, -7665045.... \n", + "1 (-7665045.582235853, 993342.4642358534, -75464... \n", + "2 (-7546437.526804707, 993342.4642358534, -74278... \n", + "3 (-7427829.47137356, 993342.4642358534, -730922... \n", + "4 (-7309221.415942414, 993342.4642358534, -71906... \n", + "\n", + " tile \n", + "0 Tile(dimensions=[256, 256], cell_type=CellType... \n", + "1 Tile(dimensions=[256, 256], cell_type=CellType... \n", + "2 Tile(dimensions=[257, 256], cell_type=CellType... \n", + "3 Tile(dimensions=[256, 256], cell_type=CellType... \n", + "4 Tile(dimensions=[257, 256], cell_type=CellType... " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#Look at a sample\n", + "pandas_df = df.select(\n", + " df.proj_raster_path,\n", + " rf_extent(df.proj_raster).alias('extent'),\n", + " rf_tile(df.proj_raster).alias('tile'),\n", + ").limit(5).toPandas()\n", + "pandas_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/rf-notebook/src/main/notebooks/pretty_rendering_rf_types.tile.ipynb b/rf-notebook/src/main/notebooks/pretty_rendering_rf_types.tile.ipynb new file mode 100644 index 000000000..03d645546 --- /dev/null +++ b/rf-notebook/src/main/notebooks/pretty_rendering_rf_types.tile.ipynb @@ -0,0 +1,613 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Pretty rendering of `rf_type.Tile` " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup Spark Environment\n", + "\n", + "Minimal imports" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "from pyspark.sql.functions import col\n", + "from pyrasterframes.rasterfunctions import rf_crs, rf_extent, rf_tile\n", + "\n", + "spark = pyrasterframes.get_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read an EO raster source " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "uri = 'https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/' \\\n", + " 'MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF'\n", + "\n", + "# here we flatten the projected raster structure \n", + "df = spark.read.raster(uri) \\\n", + " .withColumn('tile', rf_tile('proj_raster')) \\\n", + " .withColumn('crs', rf_crs(col('proj_raster'))) \\\n", + " .withColumn('ext', rf_extent(col('proj_raster'))) \\\n", + " .drop('proj_raster')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n", + " |-- proj_raster_path: string (nullable = false)\n", + " |-- tile: tile (nullable = true)\n", + " |-- crs: struct (nullable = true)\n", + " | |-- crsProj4: string (nullable = false)\n", + " |-- ext: struct (nullable = true)\n", + " | |-- xmin: double (nullable = false)\n", + " | |-- ymin: double (nullable = false)\n", + " | |-- xmax: double (nullable = false)\n", + " | |-- ymax: double (nullable = false)\n", + "\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Rendering of python object in Jupyter / IPython \n", + "\n", + "A `pyrasterframes.rf_types.Tile` will automatically render nicely in Jupyter or IPython.\n", + "\n", + "A `pandas.DataFrame` containing a `Tile` column will automatically render nicely in Jupyter or IPython." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "tile = df.select(df.tile).first()['tile']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdUAAAHVCAYAAACnuWH3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9ebBlVXk3/Ftrj2e8U3ff7qabbgGFF0SBinx8xlfbWIqCpVJGBY2CIGVMNIlKMIVTIxolKmpZlTiAaKKViFTAISKKgKLiV1GciAoy9ADdze07n3EPaz3fH2vah9s0g/d9FXr9qrr63DPsvfbaa6/neX7PxIiI4OHh4eHh4fF7g/+hB+Dh4eHh4fFEgReqHh4eHh4eqwQvVD08PDw8PFYJXqh6eHh4eHisErxQ9fDw8PDwWCV4oerh4eHh4bFK8ELVw8PDw8NjleCFqoeHh4eHxyrBC1UPDw8PD49VgheqHh4eHh4eqwQvVD08PDw8PFYJXqh6eHh4eHisErxQ9fDw8PDwWCV4oerh4eHh4bFK8ELVw8PDw8NjleCFqoeHh4eHxyrBC1UPDw8PD49VgheqHh4eHh4eqwQvVD08PDw8PFYJXqh6eHh4eHisErxQ9fDw8PDwWCV4oerh4eHh4bFK8ELVw8PDw8NjleCFqoeHh4eHxyrBC1UPDw8PD49VgheqHh4eHh4eqwQvVD08PDw8PFYJXqh6eHh4eHisErxQ9fDw8PDwWCV4oerh4eHh4bFK8ELVw8PDw8NjleCFqoeHh4eHxyrBC1UPDw8PD49VgheqHh4eHh4eqwQvVD08PDw8PFYJXqh6HBDnnHMOGGNgjOGpT32qfX95eRkf+MAHsG3bNqxfvx7NZhPHH388Lr30UgyHw5Fj7Nixwx7jwf/+4z/+Y8U5iQhXXnklTj75ZDQaDbTbbZx00kn46le/+piu4cYbb8S5556LY445Bo1GA4cddhhe+tKX4qc//elBr7f675hjjjngsXfu3Ilzzz0XGzduRJIkOOyww3DGGWeMfOfjH//4yLFmZ2cf9TXcfPPNYIzh5ptvftS/7ff72L59+wF/2+l0cOGFF+IFL3gB1q5dC8YYtm/f/pDHKooCl112GY4//njUajWMj4/jmc98Jn70ox896nEdDNu2bcO2bdvs371eD2eeeSaOPvpotFotNBoNHHfccXj/+9+PXq838tv//M//xFlnnYWjjjoKtVoNW7duxWte8xr87ne/G/newdYlYwwvfOELV4zr9ttvxyte8QqsXbsWSZJg69at+Ku/+quR72zduvUhj5mm6epNkscfNcI/9AA8/nixfv16XHPNNajX6/a9Xbt24eMf/zhe+9rX4m1vexuazSZuueUWbN++Hd/5znfwne98B4yxkeO85S1vwatf/eqR95785CevON+b3vQmfP7zn8db3/pWfPCDH0RZlvjVr36Ffr//mMb/L//yL5ibm8Pf/u3f4thjj8X+/fvx0Y9+FKeccgquv/56/Nmf/dnI92u1Gm688cYV7z0Yt99+O7Zt24YjjjgCH/nIR7Bp0ybs3bsX119//cj3zjzzTJxyyim4/PLLccUVVzymazjppJNw66234thjj33Uv+33+7j44osBYERQAcDc3Bw+85nP4OlPfzpe9rKX4fLLL3/I4wghcMYZZ+AHP/gBLrzwQjzzmc9Er9fDT3/60xWCbbVRFAWICG9729vwpCc9CZxzfP/738f73vc+3Hzzzbjhhhvsdy+99FKsX78e73znO3HEEUdg9+7d+Md//EecdNJJ+PGPf4zjjjsOALBhwwbceuutK8517bXX4tJLL12hHN100004/fTT8b//9//Gpz71KaxZswa7du3Cz372s5HvXXPNNciybOS9Xbt24VWvetWKY3o8gUEeHgfA2WefTVu2bFnxfrfbpW63u+L9D3/4wwSAbrnlFvvevffeSwDowx/+8MOe75prriEA9OUvf/n3GncVDzzwwIr3Op0OTU9P0/Oe97yR988++2xqNBoPe0wpJZ1wwgl0wgkn0HA4fETjeO9730sAaP/+/Y9s4KuE/fv3EwB673vfu+IzKSVJKR/2e0REH/vYx4hzTrfeeuv/wdEqPOc5z6HnPOc5D/u9Cy+8kADQ3Xffbd870P2+//77KYoiOu+88x72mNu2baN6vU5LS0v2vV6vRxs2bKDTTz/dztejwfbt2wkA3XDDDY/6tx6PT3j61+NRodFooNForHj/5JNPBgDs3r37MR33E5/4BLZu3YpXvvKVv9f4qli3bt2K95rNJo499tjHPM7vf//7+PnPf46/+7u/Q5Ikv+8QHxYHon/POeccNJtN3HXXXTjttNPQbDaxefNmvP3tb7eW0o4dO7B27VoAwMUXX2xpyHPOOQcA7N+PBJ/4xCfw7Gc/G6eccspBv/f5z38ejDHs2LHjYa+BiPBP//RP2LJlC9I0xUknnYTrrrvuEY0HgL22MHRk24Hu98aNG7Fp06aHvd933303vve97+GVr3wl2u22ff8rX/kK9u7di7//+79/xPNlQNqdccQRR6xgRTyeuPBC1WNVYGhTQ7FV8aEPfQhxHKNer+NZz3oWvva1r418XpYlbr31Vpx44om47LLLsGXLFgRBYOlVIlq1cS4tLeG222474DgHgwHWr1+PIAiwadMmvPnNb8b8/PzId77//e8DAFqtFk477TSkaYpms4kXv/jF+O1vf7tq43w4FEWBl7zkJXje856Hr371qzj33HPxsY99DJdeeikARXF+61vfAgCcd955uPXWW3Hrrbfi3e9+96M6z+7du7Fjxw4cf/zxuOiiizA9PY0wDHHcccfhC1/4wmMe/8UXX4x3vOMdeP7zn49rr70Wb3rTm3D++efjjjvuOOD3iQhlWWJ5eRnf+ta38NGPfhRnnXUWDj/88IOe55577sHOnTsPeL+r+NznPgciwhve8IaR9839FkLgWc96FuI4xsTEBM466yzs2bPnoMe84YYbrO/90Qpkj8cx/qB2sscfLR6K/j0QfvGLX1CtVqMzzjhj5P09e/bQ+eefT1dddRXdcsst9KUvfYlOOeUUAkCf/exn7ff27t1LAKjdbtOmTZvoC1/4An33u9+lv/zLvyQAdNFFF63adb3mNa+hMAzpJz/5ycj7l112GV122WX07W9/m7797W/TO9/5TqrX63TMMcdQp9Ox33vjG99ox3reeefRDTfcQP/2b/9GW7ZsoTVr1tCePXtWnPP3oX9vuukmAkA33XSTfe/ss88mAHTVVVeNfPe0006jo48+2v79cLTuI/nerbfeaq/32GOPpauuuoquv/56+vM//3MCQJ/5zGfsd6+88koCQPfee+9Br2FhYYHSNF2xXn74wx8SgAPSv//+7/9OAOy/17/+9VQUxUGvqygK2rZtG7Xbbdq1a9dDfq8sSzrssMPomGOOWfHZqaeeSgBofHycLrzwQrrxxhvpU5/6FE1NTdFRRx1FvV7vIY/7qle9ioIgoPvuu++g4/R4YsELVY8D4pEK1XvvvZc2b95MT3nKU2hubu5hv5/nOZ144ok0NTVlN8X777/fbpYP9tu97GUvozRNRwTbY8W73vUuAkCf/OQnH9H3r776agJAl112mX3v/PPPJwB06qmnjnz3Zz/7GQGgd77znSuO839CqDLGaDAYjHz3H/7hHyhNU/v3aghVI+jiOKYdO3bY96WUdNJJJ9GmTZvse49UqH7zm98kAHT11VevON+WLVsOKFTn5+fpv//7v+nGG2+kD3zgA9Rut+klL3kJCSEOeE1SSnrd615HQRDQtddee9Dr/8Y3vvGQvv/nP//5BIDe+MY3jrx/7bXXrlAOq5ibm6MkSej0008/6Lk9nnjw9K/HY8bOnTvx3Oc+F2EY4rvf/S4mJycf9jdRFOFVr3oV5ubmbKrDxMQEGGNot9sr/HYvetGLMBwO8etf//r3GuvFF1+M97///fjABz6AN7/5zY/oN2eccQYajQZ+/OMf2/empqYAAKeeeurId0844QRs2LABt9122+81zkeKer2+Ik0jSZIVaU2/L8z1HnPMMdiyZYt9nzGGU089Fffddx9mZmYe1THn5uYAqOjyB+NA7wFqjfzJn/wJnvvc5+Kiiy7CZz7zGXzta187YLoVaRr3i1/8Ij7/+c/jpS996UHHc8UVVyCKIrzuda9b8dlD3e9TTz0VjLGHvN9f/OIXkWXZCjrZ44kPL1Q9HhN27tyJbdu2gYhw0003YdOmTY/4t6R9pJyr5Ver1Q6YYnOg7z4WXHzxxdi+fTu2b9+Oiy666FH9lohGzv20pz3tEX/3iYAjjzxyJKWqigffGyPkH5xW8uD8XCOo9u3bt+KYB3rvQDCBcXfeeeeKMb3hDW/AlVdeicsvvxx/8Rd/cdDjzMzM4Bvf+AZe8pKXHDDQ6WD3G3jodXnFFVdgenoaL37xiw/6e48nHp5YO4DH/xXs2rUL27ZtgxACN95444gF83AoigJf/vKXsWbNGhx11FH2/Ze//OVYXl5eUUzgm9/8JprN5sMGmjwULrnkEmzfvh3vete78N73vvdR/fbqq69Gv98fsZ5f9KIXoV6vr4hUve2227Bv376HjZD9vwkTnTwYDB7zMcIwxEtf+lL85je/GYnqJSJ861vfwpFHHok1a9YAUMUPAOCXv/zlyDEeHJh2yimnIE1TfOlLXxp5/0c/+hF27tz5iMZ10003AcDIGiIinH/++bjyyivx6U9/Gq9//esf9jj/+q//iqIocN555x3w8zPOOAOMsRX3+7rrrgMRHfB+/+QnP8Evf/lLnH322SPRyR6HBvwd93hUmJmZwXOf+1zs3bsXV1xxBWZmZkbov02bNlmr9W1vexuKosCf/umfYv369di9ezc++clP4uc//zmuvPJKBEFgf3fBBRfgS1/6El7xilfgkksuwaZNm3D11Vfja1/7Gj7ykY+MFGE455xz8IUvfAH33nuv3cgPhI9+9KN4z3vegxe+8IU4/fTTR2hcAHZD3LlzJ1796lfjzDPPxFFHHQXGGL73ve/h4x//OI477rgRCm98fBzve9/7cMEFF+Ccc87BWWedhX379uHd7343Dj/88BVVdh4K27dvx8UXX4ybbrppRWGG1UKr1cKWLVvw1a9+Fc973vMwOTmJNWvW2Dm77rrr0Ov10Ol0AAC//vWvcfXVVwMATjvtNGuhXnLJJbjuuuvwwhe+ENu3b0e73cbll1+OX/ziF7jqqqvs+Z7xjGfg6KOPxgUXXICyLDExMYFrrrkGP/jBD0bGNTExgQsuuADvf//78YY3vAGveMUrsHv3bmzfvn0F/fvpT38at9xyC17wghdg8+bN6PV6uOWWW/DJT34Sz3zmM0eo3b/5m7/BFVdcgXPPPRfHH3/8yP1OkgQnnnjiijm64oorsHnz5hX0rsExxxyDv/7rv8Y///M/o9Vq4UUvehHuvPNOvOtd78KJJ554wBQwU+jjoQS1xxMcfyBfrscfOR4qUMkEnTzUv2qwyxVXXEEnn3wyTU5OUhiGNDExQaeeeipdf/31Bzznrl276Mwzz6SJiQmK45ie9rSn0ec+97kV33v5y19OtVqNFhYWDnoNz3nOcw46VoP5+Xk644wzaOvWrVSr1SiOY3ryk59MF154IS0uLh7w2J/97GfpqU99KsVxTFNTU/Sa17yGdu/efcDvHihQ6e1vfzsxxug3v/nNQa/hoQKVDlSowpynihtuuIFOPPFESpKEANDZZ59tP9uyZctDzs2Dg41+9atf0emnn06tVovSNKVTTjmFvv71r68Yw5133kkveMELqN1u09q1a+ktb3kL/dd//deKa5BS0gc/+EHavHmzvddf//rXVxR/+OEPf0gvfvGLaePGjRTHMdXrdXr6059Ol1xyyYrI24Ndz4HWsgnCes973rNy4isoy5I+9KEP0VFHHUVRFNGGDRvoTW960wHXX7/fp7GxMXr2s5990GN6PHHBiFYxCdDjCYNzzjkHN998M+666y4wxkasyj801q9fj9e+9rX48Ic//IceykFBRBBC4H3vex8uueQS7N+/31KlJ598MrZs2YKvfOUrf+BRenh4rCY8/evxkNi5cyeiKMJxxx2H22+//Q89HADA//zP/6Df7+Md73jHH3ooD4tPfOITeOtb37ri/eXlZfziF7/4vYoneHh4/HHCW6oeB8SOHTts1GatVnvMgUKHMmZmZrBr1y779wknnOADVzw8nuDwQtXDw8PDw2OV4FNqPDw8PDw8VgleqHp4eHh4eKwSvFD18PDw8PBYJRxSURNSSuzZswetVsu3YvLw8PD4IwMRodPpYOPGjY/bkp+HlFDds2cPNm/e/IcehoeHh4fHQbB79+5HVU/8jwmHlFBttVoAgCPe/B4EiSr+zaT6TIYAhUDYV3+HfcD08JGxO0bUI0Rdsr8DYEl0AlCmDEVDWcH5GEChCq5mJUPZlva1jAky1n83S4RRCQAoFlPwjCPsq2NQABRjAgAQTwywfryDfqEGNCxCZEN1C6fGeji8tQgA+M3sNHq72gg76hhFW4JPqSLnzUaGvAwwWNQdThjAeqqwQ9TlaOoMEBkARZMhNE1PCMhbsHNGAVC01LWJhgRFekICQjwTIZ3R526o7wKAqBFEqouwFwxMqvcAgMZzUKEmkg0DpHsDpPPqMxk5ViHICDLSx4sYwNW9M9fSP0zNFcUSQS9AMquOKRJAJvp4IcB0zDttHIAzQhCp39WSAoNMnaDIIoiCg3XUCfhUBs7VdRbDCMnOxI6LYhoZB9fzxgsGJoB8Qp0w7DJQoOegZAgygOv682XdzRVxgAn9OlTzX7akPb6cytV8zCRIH2DIptQxi3UFkvvU+ogXgd6JA4SJWlskOCSpuUzSEkXO7bWwboiwr/6mCokjI4JsCPB+4M5d0wOLJKCPETYLBJFApOdx0I8R7FSlJVnOkK0rnbOJEVimi/A/EKhnzVx34J67cKCuG1DPIM/V+QFAxIDQlStZCfDSHUMG7v4y6a5HJOr9eNnNsbkvRUuCIkK4pA5CEdlzgxiCHJW/1X0FgGAAJEvqgyDTz4V+/oeTgNTr3TwzZv1Hixzxop7GASFv6uelqe67uRYmYNc7L9W+ZPYeCoBSz0E+KREM1THiBaY+q+t1ljPUZvXaDxjyMffMMOnmGwwgTna/K1sC3NzfDkOk95MgB0Tq5jXqu3Ewod43e4MaN3P3Sbh7JFIC6WuTESHscMhsiHs/9j67Vz8ecUgJVUP5JpQioBTEgLKhP4uB2hwQDrSgSJldeJyAeFm9H/cqmyfUhs+kW7BBCGRr1Xl4RGBCvZZ1AteLVTYJjBF4bHZ2QGohmvY5eMlQNtRn6VOWcdI617ljpt9CVKoBDAcJTnuK6tIxFfVw69yT1Pt71yHtc+Rr1NMXCkDG+imtMeT3txGbDXTTELKlxzgX242WOEARoPcOyNgJRFYyhD2GmhEcy0ogAEA2KRERB9PzGkZApjcu0RZgUisLDJCMwLQgjeZrbmMNCXIdQ3djRQAruYC4wxDo+vAhgLwNu1nLiJDqMUUzSmiXTdh7YfeOkiE7Un2x3gwwHEbQzzriZg+UqPkV/QTUD8GmteDPY4hcjT/MOYKUW8HHhNtzWek2NDTUBoKmvhfDwClkoRq6Ybk4gFKvESat/LDXHvXU/8NpCZ5ppbDGUG4EGrN6zc3VEORaUNQZWB6hDPUgcw6mb2i+zIC1SponNYkiiiHG9PkyjmCo10FC4JEE02uEGiW4eQAIqE+bYv0BsizFYF4pGtECR6DvNWKACkI5ZmYZSBfUMVgdkA0g6qr3pVYgACDiBJHo6yIg4ITBtPq7bBACfa/jRQbGKkpwUFHkUrL3Qq7LEe5JwHOjbALUNIqtBM84eN0os+45ZwJApAUrgGDIEOhL4QxAW3+vR6AYoDEtRNoEpscRLzHwAUCV3gZMy41sCggK9TqRSmCZdQVeUQAj9RsjzLIpCan3EC4Y6vvVBMi6UjqMnltbJkCvKzHGgAYAs/cIOKOAq+ectPLZ2BtA6NeoARSrE2ep2guMwioTgOljDNdKq0gCQDhgTpmKMfKMh4K5h6YAUOlk+Hh2zx1SQtVAhgAL9f964SWLaiELbRUxQQj1QxR3CVxvbCICiDuBSxxgepUTB/Ixt1BY6RYUKxlCoT4oudI+gx63n5lxMAmUqduAto4tY2NtCQCwVNSw0K+h21GrLwgl7liaBgBMpT17fcSBfFzazaOcKtBuq6d5eW8LQZ9D6M2EOpH9XTRgVjgyoTYuu+grFmG6wBD1gGCohQ1nypoCEC9yyMhtcCJ1D1LYDUDcTA7AhBNKZYNAbb2zCAbBAeg5EQVHZCwI7gS4SNUmYL4X5MxuzsGAIFInjDkxy0IM1hGorwbclwzN8T4GfSUMiBh6+rXoh0jHMsSaRVieb1glgAl1r7gesgzdZleOEaRmKChQ15ruc2UepTFwSVlPRlmJF5nV5InBStWwr67VWFUUEsiwHIMQ8RJDoq36oCC7hot1ACsYSCtKiCWCJWP2AEbmBaFEwcmekBJplQxwAAE5hWE5sqwEa5bItFUvFxLEs4EVdFxU1kCgLD+uBXW84O67um9u7njpBIpIGITeaIMMyCaZFZBRR1n5gD4PcxZd0SSrlMpUorZW3XgpGahM0DtcjV/WBPhA3ZdwWY3dKszEkHScAshzZ2WJ2ClNwYAh0uuqrKvxGiWSAnct+TiBCVgGqmqFB7k7r0yAsOeuRTFo5jkDoi4bObexTpMF5qxFUmsmHLh5LJpubqR75EckABNAvMwtc8III0pwPumYNj5kFa1PCXh7HMKIomturoydghP2GfKWBM/08UNCMGQg+fgvm/D49AR7eHh4eHj8EeKQtFTLGkCpsjKspZQCPHL+rXgZlkbjBVm/XpkyiJqzRosmRjU/5o7JpHstY0WNAADVtFaXhfZ7MtLaKGNKq28qE6iTJfjVwkYAQMgkhlmERktTl3GBHbOTAIA7FzZaPxUCAoWE2lZlvcpejM7OMTM89b+hMYdOr8qnhLKuAYCANUfMo58p3mjQj8EfUCZW2FfXVbTcd6U2xIq2RJAx+5lICDJV18uH3Fq0YY8pmm7LQF8boczMhwQeC5CmzikL7Pz0N0lQoo/XC5TVo6ezbBDyMWPJcGvJAErLF9pClAkhGldz2Kxn6A0SMO0rHeQRhPZTR40CWT9C1q3beQ2X1XzxXFnBxhqjULEDACCbwvLmtT0hyjqhaBvnoLM8ZKTWYLyoDlI2CcFA052Fo0TLpraOxrTZLRmiWTXGdJYh6hG4YUFShiBTr5MlBjCOom0sBW7V6HKsRHK3Mm2KpkSo/fwAIOsCVBf2XOCEdFLNV5GHIOPTExw0o0zJqMcQDF1MAuAYhaJFIK78yYDyoxsWJezrOTEUeD56z6w1qundaNn59awvMASKGpBNG55cIqxpP3LJMdynfBHxXABw5SMGABZJ8EU1j2FPWV5Mc63BsMIySaj7p29h1QqnoMJQNIDBtIQYt/QIWNccX1nn5nfZBFkGhxcVC7anWQm9XmRNWkaLa7dCOHDWo5kfGan1A2iLs2CVc7k5ZVKxBtI8Qxm36y9eUq4G84wWdXfMYlwg7Di/OhPuXpUNsn5T6D3PXA9Yxf0zWQD6e0XEwOoCpWYK6rtDhANAZI9f2tfgkBSqvHQC1QaDcCVQo56j0QzKGkehN4jhWhUEYHwNwYC5YBuuFpqhTMymCCgaJ9QPB9YNICWHTNTJRcEB4QRUY20fa1vqx8eN70Ohn447FqchygC9XC3ETr+JoKWE78TGJaxrqt+kQQnOJO6aVx1RaCG2NBEFBDmdgYfqoSr3p6CW2gTqYwOksTredLOLnfMTGO5Sjp/mLkfZpfMSvASKmvbPtZkLTGCGztSbwtocoQ5eKXoRmA54ySckwvHcCrMyC0FDR5GKouJMYoRkTtO/AVA0tb8vJAynS0AH/rCCW2WBuNp0zCHKOgGHKwF+1PQsBqUORpIchQjQXdQCZjG1cyqJgc/GSOYc5WsEMy/UZmI2HVkTdhzBUghR1wFNTVKKjJseS/Up2tZRnFQw62bghQuQy8cI+ZRAuKSFfYfZDS1eVpuzUSzijgRxLXgyAgXMUtEgWPo9vT+y94yVSqCQVlx4o0Sg/bBhKME5YdBTg5HdyG6uUYc52rZQwsZS2xVBGQxV7EExppXKiJSAg1JKwz5DvKTnR5K9Z+EAKPUaK1qKPrR0IYedt3xMghJCfa1SIqeafexfVnynWEisIkQBoZiQ4KmeZLjgNQzUtZhnV9SdW4FC81zDzpcRZuGgEgilg3eYcWZK5gRiDjDBrCAFVRSEzB07bwPFmHTxHANu7zWTbIQ255UgprJB1tVRm1GKVaGDn3ju3BRlg4GVBBZq2niWW1+xDACZOqE6eFJulcNwIXT3pccgEsJgWtPBshKMJLRCcgC/b7AY2rliBMiSIZ1XExIMR/fjxzMOSaEaDNX+V40YDDIg6pK1TkXMMFxrhIbTxIg7DRNQlpmJ8KNIgg0DRKrnMwbTZBdUtMRRjOsVs5iC1UqMjSu1Pi8DDLSfdHyyi3XNLgpt+t2xvA4PdNQG0ZlrgPUD60/bvHUWT53cCwD45dxG7F1WERPrWl0ETKKzoDWBpsDEkfMAgDX1HtrxELc/sAEA0KtHYHqjlZKjlagnbO9yG8UdbTQfUNfdul/YB1/EylI3igcXzFlYOUM+pgQ3ADBOKDpqQw6WQkhtpdene+Cc0O/qXbgTIepwfS+U38g8YPW9zAaQZWPMCg0ZAjwPrFIjU+n8VEMg7gDZuPq7HBeoJ2pnWcpSCKmtQ8ExHMR2DiiUEJmJcmEIC2Y3Lsbd5idSfX69ycuUIayr44djGcScEtJlWwCJtBHWQZ9bi40XzB4HUEqdCcKiAFg+2lk8INio4bIGwPRsZ8pXPIidIB2sM4KIwAQh1QEsokZ2U6NKhKxMSFve+lp6IWSkxlsOAhfZDWXtxcvOmjAMBTN6lBEwCaxfkx3ZQ6uW2Tnv3Neu+NkZ0vlKNL2s+EZbzCoxFOjjGZ9ql0MaZYoAtHOsbbm4gvJe9Vhg3hsAACAASURBVMzUK0pLNiVAjBCYeRyESOe5PX7Rkk6Pq8RDBEMl0Ixwk2FFuaqwFXlb7Q3GXw+CC87qBOD5aLCfASNYn6eMCaxwflpjlQL6+hlA0r1njgemhCmg1kA+xqwgDTI3p6KmjpHMBvaYJoo67DIVBLdOO8YHIeL9oZ0fY0ioCG0a8anauAYtUG3kccGs855Cd59ZwRBVri2bVHMjh9W0iscnvE/Vw8PDw8NjlXBIWqoGxICwbzQqRXsM2krPGK4lS/FEXRdFKhKGok3WamCSId2n8/2afCQXTTSlNQfKBrPU0vQRsxhLhpgfKEuyN4ixYf0CAKAeKfWyFamTb6wvoRaq9361rwVqCKQt9dn/mtiHHd1Jez01Td02whwTSR/71yhfUiECLPeUuj6/1ICcTcC1/5Ul0l5LEEjct185YEQnQkxAODTpGU7/MtSqsQD60xWqvEmQYyWStLDHNG6SeNLRy1ONPuZ6dciBWoLJPLcaPxNKa461xc8kbFRpmLjzhrminY2PLGwUlk4W04TF5cRScUmtgNSWUi+LwfV94YwQhAKi1PMRkqWhWb0EDi8w1D5W5BzRgotClhFBapo3aOcotYVbdiMw7dNjnMDvT611Fy8DmU5dKdqko8f19fQZMkNpTlZ4MAL4IEDYNRNEdi2BlL/VRKYuHkkgfcBkTlF7xqpikilaEwAF0loQYY8pK0H72Vjqzk2cI1gOXA7kMhuJ1A2qwwxUbII6CFCsV/c65YTlTh2ip/2LXW6fpyB/UGxDXUfQm+vW1tZggwCFZPMmRUyWpgSAMBbY31HrvT9Xt+kk2aRjiyggICRI7QeOO8z66otJ/RyY5zqS4HPKvGMDPhKRWzacX7NoOQuuHBMqHkCzEtV8UxkSWMmQzBs2xo29rLljU6CtU2OAVlwOJj/WuAiKivshXuDWly5SFQlMxiVDDEXbUbXRMreWZNkSNrK+bBBovIDQaVHxgnvmZUQ2N1fG2keu/abBwNHhTKprK7Q7q6wTuIl5qJP1xyMEpHTzIGLSqTjep/q4hOHueaHC4AHto4ldMYJk3tEn4YAqKSIMUY/Z3yn6RD+IUM+CaOlNvlVAmjxVBqzfrCjYqVof98xOocj1JhMJxHp3KiVHJkOEeiX+emE99i3qhLZYoj42sDTXTx7YjEKoB+LFW/7HCor5ooHFvIZY+8VKEUDqDVTOJ6CGgIjMTk5ggTpXb7aOUAduxDlQ38cQd7TPM2EjtCJYJQ2icClBMhHYfNicHUspOUI93qVBilzn2O6eG0fWTWyKikh0sAjUceMl2E2iNucoIV4yEDNKjN7YG+pGMS6t4BQlQ218iESnwwyyCKbJYZ67ZV/mAUhwNMYU7yoEx0BLbSo4ypwDeoxBj9tNsmwKnWSqN7JOhNpuvQkLQEaRHq8KxjH+0WzM0aJhjyGflOAm8ISTpQVZwRD03QYcVShXMLc24yXCcA3D8v/Sc5AKtH+ix8+BwXrnG5SR85sGfWbdEbShAGNAPVX85kRjgMW+4gR7g3DEZysSWIFYpXuJAYO1ZMclaoRA56LK+yOwxG02DK54AhPKJ29+xwuy6UE2RU1/n0pmf1elHLlkyBcTFFox4gWrUJVkYxni+XAkfSWbktYdgUTYQhYAwOYj69IIB2qvsL7vgOxrXikKwYccsiEgm85ny7SixXKV/mIEYpVCtjnNUMqRqLmANSZhC7CUXAnbomEUdbKuhGTR7WVMqHgOCk2woBOQ5jqMLzlaCuzaytZI8FkXdUmBCoQDgDJzc8oLhqjrlCswFTtgPquCkbs+St0c8wFDkLvvpvuVUv1ECFTy9K+Hh4eHh8cq4ZC0VGUMsFhRTS4wwQQrGc3YaY/VwCRWAqIOlMY6LZkL+ABBTLpoVLE/tZYMIsK++xW1OpO0QYJjao3iN588sR/7hypS4a7fbVBVb3RE7ob1C9gwoeqqxVMCuQywPFQq7vxMG4G2LP/jvv8X4Rplba2f6ODo8Rk0Y2V5zP1inbUCy6OGqvqMSb8RzNKd0UKIaMlZ4DIEZGisU3ed8TIbKZwBctYXbxdY6Newvq2urZAcSwOl1mdFiMGyes2XQ3Bymm1txkVGMgFEfVUOElCUe9zV6Sohs5S0qDFgwCDuU1aViFz6DhIJpIUtyxdFAnGo5rQ/TJD1lUYeRBLTaxeQa4t//55xBJoOY5KhHCsRjKl5DKcEGjXFVy0sNMFDCbGsTNB0bzgSuWhe81JZiyagJxyowCtARZqLGWatsWraRn0vd4U4JADmqFDQaMBU90kluKabwx0pikZlHKWzkk3AE1CJegWQ1nK0ahmOm1SVu+5YXIfhQAeXdVWFL0PLBUPYYwCK3QGA/lG5oif363kdMLvmRF2f1/yuQvupNBHHSgSZC4AjBhvBykpVNckEC/HCWbfDdboWoY3kJxv0Fna5LYIQDFVqXH+DDmyrC5uGRoyDldymfAQ5G7G6ilbFonTsLEQCW75PxgRWtXZLZtOnwPQ9M1RoCyh1AZag79Z+WVfWZzUoytw/nuv0JBNcOWRIFg3N6izfqKPOZ9ZcNfpXRmqc1hIuHTvHhLJGjTUddVygmIzcMx4vcR3R7cZoIOqEfJxAsVskpooXXw5H9gxVvEJfi7mvFVr88YpDUqhWYRZv2FfpB7w0vtJKHlx1E2uoxcsNrRuRK/22pgRLBAKdz8lzJ5RE06U6yEGIyQ1LNifursU11ufZXt/BYBCj0H68fhZjWVOa3dmG2hj1udmQo3GfqcoE9CbV6/WNZdy1vAY77lLVltKhS3mhLADLuaXEypZAMK5TSFoF0pYKO5y/fxxMBjalIeyrHDqDao1WGVdon/0JRFpgRkcsS2LIdP6pKAME8zotpMshQ7K+xmDg8oJNRaZs3NDBzNZG5RXBFQxViTtbO3dCWCWGBRJ5HqI0lCAnWzVJDAJwnc40Nd5FZ5igs6zr1PYDF8HakJjevGDPt9SrYXGpYeeR358gGjo6UtoNyKV+5DW1SZr0KiZc2pYqk+lKUuZrSku/E4fNbQ0Gim4zgocCt6t3N6tUIlrQQjB39ZqNwHP1c8nVoq4JpA21+Ne2ejh2Yh/uWFoHALj/njUIBq7aV9h1qRymRra5zsaJc2rsnTrYrpoVpOp8+r7UCLUZJ2yKpnOn5C1Vos9s5EHhNvWi5iKvw74SNGbzLVOX920VqQpsOk/ujl3WGYZrCcWkeoMPAkuHm/QX69LIna/UKDvGj10VtkElgpUJBgrdWChyfut4Xgl3o/DkE2KEOq/61YN8lCY2e1Q+ru6fcRHESy7WgzGgMeeU0Lzl1iNx54MX2h9q4j6CzAlwkMqZNudmVHnGI5dbnM4CTFR8rJFyiwFA0RKggBB0nF/Zzk9FGQv7usKZmf5A0dzkmPPHLTz96+Hh4eHhsUo4JC1V0gW3w76zjsAUBVWt5VrVrKoBTVwApYn4axDGjlABSCkx9AYxyvX6N/3Q0j2s4Fh7mGpL8dyNv0Od57ijqyzJrfU59LVa/91dT4G8r45AW1yd+XGr7fJI5RmayEOeMav5Dg4rcdS0shr29dpYU+sifIrKYZ3fVMPCgk6Ey1Seq9AFDtJ6gbGG4scWOnV0utpiq5Uo6xzxkqMqjVZJobJAqVJJygQqUEwYzNRh6oZHy4HtsoOQEGjtPJuQqpKLmX4azf/N2y6/UEZOO1cRoPr6ddeLYkLnvk72ketatGUWoFbLMVFXI1kcpDYwDExFAwNAITiaaYZTNu4AoBoT/HxRtZwqZIBOlqCfq2MO99cAnSMMXTTc0pGVKM2gcFR5kKnX1eo5nS3aip0UoMq1sZIjndMTJB0bYihXcy4KXGI/oAJATGEI4i4CV0basgxMYBS3VlpUK7BpUq3HsXiA/2/f4Zh/QOU58yF39HBKCAeBC9SrE6QporF+Pzq5uuilPSrC2dB5quOOOkZ9D0e2pmIh1kvkOpeT5VxZ9ZqNiXqEoWYoOk9yxSpUh5oHWcHGJOCkuugY0qB0zJKMnMU+nJagiRzQEeeyWapC3ABCHbBjrGuZuPXOSlWdyBV/UPcVUJbYYKMO+KoJsG5orVeZkOvsMlDPqs0rDWCjtCEZkhmXL0vcVVTihaOQmQTCjutuU82RZQTkpulBqI4jK9SzqNCx6vyOFbLPU64OVK2qZuYgXmZIFvRvQlXMw0Zs11xFuLDDdU61G6Orceys4iCH7WgDaNo/JyB/0DgfhzgkhWq8RAhiRfcaelM9MOSov7jSfSZ0xQcAhjJ0VFAwAJZ/p9NQGhJhO3eLqJ3bUntJrbDpMjv7k3ig38JQR8L+984tKPWDHiyGqtybpt/CgUsMLxqqmEDRVqu5PdFHoKOEx6ISexZ18Yd2F/PDhvUnxqFArFNcCkaYnOjhyIlZAMBynmLXghq/FNxGCZNQFLENla/QOCJQAs9WtGk7Oo/lDFQjS8PKAKjdrzcx7c8BlKCoFk+nwBX9lqEO7zcysASk9WG73zCpS+HpaEvGgCjWvsVIoCwDLOgo1n43sUnzSStDu652/02tRUzEA9zTUdWnblk4EtlQCdFGc6gK7OvCHDzjCHUlIG4Ep5GB4agf1ZVtJBRNsqXg5BShNJWFAt0CbdxIS4be4eplui+whSCSBbUuzYY/nGKW+gyn+8B86nxRzBU3F03pip8DKKZKsFgNMkkK9At1nZ0sweJSAzx1bfPIRK0OOIomuYj2do6TDt8NQK2dfb9VlHF9Rm2mw7XG78ZskbDeFgFqCCRNNchqmpUQEYg7311/Hbdl9Yrx0s5v0ONAHyhtMQ8XuxDORog6TiCWDVeRqKwTso06MjqW6nk0cQ6FS5HipfMZWpCZfzU+IzhALqaibDh3TLg/UilH5llYcg0SylRVShIm2jggK1F4RQkAU+UurdAm2CphVkhqVBUG01oO0Modc75SRSHr/SozKWr6unOMVB1jlUh+84yq340qvar8oJ7G1mjJRbBKmUJUnotKVSpAj9dUreK6olP1HI9THJJC1YA4RkL5RY0ZxRXBQPnyAKX5Gf+e8SnZPDu4hyjoBCgpRqC7rciSW4sojd0mds/iFPbvHUOq+17KtgTqbrUFObP+NBkzZDpnkRoCUSPHsRseAADkMkAmdJ5nUGJDQwc0BQLLeYrZvjJjNzaXbQ3feprjyRP7saenkiV3/XYaiS4VxgMCmYo1BRuxiETshB4FOrfQdodxD3TekkAsEe9V15rMu4errAihskEQCWG4zvgNOdL9biPhOQMzwSxFJbAlA+KO3tBqAMCQ6fqqeRLalBpZcFA/sNYMbxaI9b1o1DI8sFftzjO/WzPyoMu6U5+HoUAxiKyBJFOJUFeYYUIJOTOuoNLSazANFEeoN9J6jqKXIGc6gKdfKaWYEhA6RS5MChR6s6MQiOdG/fum7VnRlCo9AUCxlCBeCKxVm60V1q8X6MAQUzIRlescDmL0tR9fZAHYIHCWU8UpRA2BaEMfR04q3zJjhJm+SvHa9btp6yvNx9TGWtvH7L2x7EWzRNLIEejUrSIPbc4qK1RAk1Ggepsk5FqlZDRamVVwRJ1D9EMrSMEJoc4jrc2ovEwbcFOpfpRNlwh0/9cwKpEtpojmtJIXOtaHuI57CN16NLtjNq5yXatlCk0VtWCgUuzM/FY7F1UtzqKl77WdWObuh1SVjABAJhJBN7AMUTCo+Id1tSIrPOOKv3jk+qGZjSrr4U7NczffIiW7dnjpfLmAtpr1kEXi9jlVMQku5zl0ucC85KrEZugEupkfUXPxDybQMagog0WL+ZQaDw8PDw8PD4dD0lKVIQOzfVPVe6LGkI2ryDZAW6e6WJEMyCYqyxAj9X/LtrT0D8USCAlCW068UaIs1BeXhnUEurA85wRIZpO45XiJSFtRfFJiOBUj3Kcsy+H60hZ451xirDnEnq6ieQd5hDHdKXxDfRmNUKl9dy6tw+6ZSUuF3tFPUU/VZ7WoxGJew867NG23L7DUWzEmrZUWDNUc2KLrplA2AJZpykprtems06bTmRBcAFHHRCKS7VhDkauVW+swDNa7ak4iIQzX6HvRkECztKk+fFjxvUrYykKqcbq0fjciZottUMaV/9NQcQ8kyJrqvmRLqYuglqp6k/WlpY4eLGdryjLSaRLh0CW8S02xmbQfVgLDNdpa3FjY+rL9uTpQMqSm1mq9Uo0r50jmOXIdRRltziD1tbHCMSVghO5mWJ4uXuIgXSTe1NzNdK9LYgRmGhNwQtmQzkISzNK6xSC0/U3DXFF+xt9HoRsjGFAWAfb3FOsRcMLsjPa9ZswWVQ+GDFO/JMtmdI6QaB2umBMCMOgnKEt1blm6+0mtEqWMEC84a0l2lQXaq1imrFC+12rhdmPNlTWVVmNqYvOMuwbjibRVfLKlFOFiaKnVeNGt22JCRbOa74qGtMU3zLhsL2Duon5VgQo9BwMGDBmkrqVb1ispXgxAyWzT8qBSVapsSReFDPVsGVq32uRDJKrfsU21Kp0Lpmy6Hr7JgnLbGCpVJs43HfaY7T8LAGFWsU7N2q7UQ69WejLjzcZdRybzO14p5EAc9v6GA5cuJAMC447yDjKXSiVDtQarrNHjFYekUAVT/8oa7EICgGQetri0LbcGl5cJAKKpqA5biSWSNrdN9aeCLfMmhwGgBaysC0SJWpXF/Q0ElVZbUT1Ho6b4qkIEyJfdbYnGMoy1FJWYlwGEZBhkSgoOlxPUdCm42WED9+ZKC+gME3AukQ3UExFGAqmuLLS+sYzb7jkc4bLe4CpVpCgmsMwIVZXqYR6WZAHglSACLuA2xsDlE4pU05Ws4hPSD6NpGQc4P3E878ZhabOMgcrIpRVEsIKBpE5HgHqQwx4D6fZRBWJHDxIDG3C7McqYbDBPtTi9CcowASXJQmjzEUWs98KaEzaFLm7FSG3qpU4lGK4l5OvVPWSRRKirWVGdge1NbeCPTMh2BardnajNUNOTg71NxDrVqTbrit8vPkWN16QwhD1XHrCsaWFo3RZcKSUAUHCEncD+TSG5oLchR9hxi58LR+9JApjOzSXJIGYTLBq/SCeySg0jIJkzaRYEkTDMn6gGVlvTx/K8yR/h4I0Cofbnil6kckIBpPdFiDpu82Yls59V54ZCCYpgff58wJ1ftqXocNsdpkmo6XShKBDICnXfB0Wg3BZ6XRGvBNRkSukzAT1Bn4/kTMpYlfcD1Do1v8umpBWcQRagbBFKTbdTTK7qU87V90wFLsA9P5UG7hTp0pX6/la7zWQTStm1ZRGn3PPIBFB7wFXgogDQhcdUcGOvEjvC7DJQ5QL1S9PSLl7W586pQi8z5EqXUo0aZCXAqRKYZ1N8jMchRqWsJXM+8qFWSG3KjnahVFr/PV7h6V8PDw8PD49VwiFpqQ7WE3iq01OGTvsv2o4Cq9JLTKgGxICyGGRDgNeNugtIrjX3gDA21ofQqllRhKhPK7Vvfn8bwc+UmVNfUEFRfZVRg6IXY2lWmchBjyMZMIz/PyoY6diJBzCdqGMIcPzXvcdhOGt6NRFm71XW6WyzjU0bVGrPcBCDdtbB9LVEmzu27dZt9xyOeEdqrWSRVLVdZim0bFJpoibgo2gAoc6H4RkgQpW4D2gKte9eA64iVdlgtklxtQh3ul8V0TfMQDbhioPznAFEtooPJ1jaLxioermAsvqqPSbDTmDPX0yo2rwu2phZzbpoOeuCEUPJyUaBVougM6l+b+vsLrjek2Dqeky0a9GWqN8T62smiFAXAIE6Rj5hOjIwhDPqe/m4BJMMyT5lpsVLlcIQHOgdpn4i64omZ9pCjDvMMimipgK+rIUgGCJt/cfLDNEyMFyjLfkxaQuVMLg5zSck0hmGVN+bvA2gr+sHM03973fUjaxQlYaKzCZVYQXD0gxm63ZOiwkBmQfIdYQ77wW2HV040FaZidxtkWMGGqWtu4xEgscCsqPmikKCMGxDO8fURA99nU5FxBBpUz4rQgx1e0G+FKrn1RTYr7BUJgWlWtDdVjIqVO1lV5kJGK6DHYfpDQvSQYzmGJWgG+LKamXa8kYsbVUz3nPrlmdcN/lWfxdNhlKzOlFHWaFmzgHX4L46XtNEwLhrzP+ACpySlfrNYaWak8Rok3mgwixVGgfwAgBnlkaXMUFYylhZrWXs1rGtllRpBiDjSuMEg9B6ZR7XOCSFquHuw0qZrGxSURFMb6CsdGH+ZYNcEfQuQ9gPIWNdJSghsCkTrThEp1uDrNDB2V71BNT3BrbSzXCKIZtyvRujmWgkV2xweIGOLu13d7AGz9h0LwDgB4tPRu+BhqWN6pN9W05u3dSyTR8J7mggWga6W9WTmsYFurq0IZ+NXa4cRulZVjDAPGA1ibAbjPg4bPRvS/lETOFzwD0gxIHafudHLVqOJop6LtpPVjpv6KlyDYxLgHHnvxQtYSrQoVjjUhhM021D4YZ9Zq8tng1G8uWCnNn+m7IhbL5pkAhIwVBoejmeCUc2i7Dvyu2Z3DpA0a5VWi0YBvZcxIByrekKrbg23jEdWlzkZTrLRxpUR11yZf/WM5s6Balo8rG7zFyraFtA+b9Y6VI3CK7iT5ApRdFulIxb32ttU8dG42b3tV1VHSjFxeQgi1hFJxv/P1glzWLo3CL9DVLlZe7WlZ0yhlznD7OCIViMbBQoK91Gy4RaV0ahKisRuCyWSMfVQ1MWAYpeZOlTqkusXa86mxMxdPqJpYbLLAS0ElwWAaDnnkJVptCWAIxh158qI+g+G2m8XagC9anO0+xtYNZFkD4Q2DWRTaoG60aIyYgcFZ9IQAKkqe1gPrL3TKTkUm0Y0Lg3tAKyv1HaLkzp/dFIWgvPGLIpoefRpU7xXFUsG4lCNootlFJse9Y2yRoWwVCd35YdJIZcd1Qq61RRmJkW7iayuRKLYSs0mRgUJ9x5CUeH67zuqmLDBUC+SfnjEyxn4FxpaEYLVIE3zFki5PLWZAi7YIMBAzhs2bm8BRQ6oKm7r6l8I5HR0gj1ve4pGE7qzYJUVwZb3CDRASUAxLRE0CzxjI27AADrkg4+d+8zAQD7d00gXAogN6mNhoipoCcA+3ZMKUECIBqqBzxYk9nviV+qpyOC2ijtA1K60P5gyFaULzMN1ykEmClTKJXv2aR4EIdVTuIl/cDqY5qSZuqYrkBCWAAycPVbox6zfk1GasMrWjrwpM9Has+6TjFSJbpXAi2MJVw0SAnBwm2M0hhbkoFr/1sQCpT9BLVdrsOMEVhMKMXLbDJVrZt4RTGA8geZNKhiqkStrYVByQFikJ26nddgSc/BUPmpTSnLMoXtBFTWyfqDow5DfYYwnNLX1oYrLqFrsBZTOlc35wi1j7k/TYh6DIm+h/2NEsFaNa5hP0Z0r5qQsRk20jWFAqc8JPNMl6Ezn7m1Q8xZL2GfoX63U1KHU25ueM4QDhyjYM4BAHmq68Vq4RC0c+uPTmKX35ENIrAsAGlLL6oVtrQnAIShtK0Pl6iGYV9JJeqFNsgwGDJVd1nPXTZR9V1C5147RdH4V8OCIcgJvQ3akh8je51MuPtOESHojdYMlvq16axkfKem1SSgUrWsorwrRJABA91OkU8PrRKZrVPpWCaegzcK2xCdJEPYcTnUyXxVcLrAKhkpJckIUlRSgExZQlvAo+HuU9iv3D9SDIlRhMIutzWISTM4RlGPhmxkjk1cRtFko7ndhXoemPepenh4eHh4eBgckpZqvKy1pkroellXfL/R2sqao0FESta/Gi8ramj5SB3hN1kAHdOZQ/lDTGQdF8xG1vKC2SRrVigt0jSiZuM5Dl+v/KFHtWcxl9Xx33tUaZ3iN2177rQGZGsE6nWlzvXma4i1Py4tKhr4JCE6ooNIW7H5D6fQ0FRt3mbgsbPIZQTr8wRzmqMr0qD/JpdAXjS0r1FbAPGSm9uyqSJ/I23Vhn2yXWVAztLLxvhoc2YOp+JJ7dPW/qJg6FIfRA3Wl2b9T5WOG+YYYZ/pkH03Njv/MoAw3Um6KWqZi3ytdukwRQlExS9rUx0EECw76y6bIlXlB8paFL9t2XNWq9bYTjMwlp6zarIJ59+KuszStvEyIRtzZRtFQraoRjZFkIcNwQ0jPoythc+EWhO9zZpxaQqEdysXQdxnNsqTl6SaPRj6nSoMBR9dq5DMzTcja6EYV0A2oS3tpousNvfB3GteuMLyxZiEHCuweaNa/5uai+iWalI7eWI7HJFkoEQA2tIr+6ktgJG0M+R5YAtFlN3IRuRHi4FlQGozBCbJsgGG3geUmyIfkyOdkgw7IuqEfuzuR9hjqBYxMT7asKtcBdVuNjb6vFTuJdP/1DQcB4CoEyJZMOcl9DYC5WGmSwS3+0u4rPr5SlNUYyFGkDk2w0bWmmIs+u9wYAqlKLdKMHT+VwqVda2ui8CFu4eywlhQ4NLrbByGuUzh0nAYYaQHrLFc1e9c4X0R62YHFctURoD0KTWPT3ABW5PSLC6RqCAF8+ADlZypgCBivRhSoHtMjkiH7BdLic1BDHtKoFnfhnALqqyR21ihc1RNOHkWYM+8omcX+zWEgURft0hrzjHk44YyIVBdYHiP2rBrS3zUP2QrtADDvQ1EWijFAijqTvDEAxfskFfajTGq1DTVgsUIGJ67uYq6ozQsUDnemBK2pk4oL9wcZG1uhXTRUt+zHTFKJ3hM9wpZaTtVbS9l6bxc0a7VMdpyqoGmSSvl3ywl1WG2+TKYol1NdZgHd+1AJUiq2olGcBrZQMumQKQ78DR3ws6pjJwvGsAIBWoqJZlybyIlO/9RV21ygPpO2VSVlAAl3M0GV9YIVHLwRR2k0y4hTB5jNxyplTz+ywhMd2EKhy5lR1WGIphduGhWArBaEhgrbM4wiKHUCzfpc+sGKZpM53abADVp1x8TDP2NwrYDY5nz97F1Q2yeWsLz1t8BABgL+9g9VP6U2xc3YlnXoqaCI27lyLua1uWE9oSSpvL2NwAAIABJREFUcnEo0M8iDPZoyRGRpUKjDnOKFQd6G11XoNoDTpkqGmRzfs29qd4rXrj1XzYqcxe5VCeQTjcxDLLECBcYLbsWetUG76x067sYZ6olnO46lO53cyVD9ZtIC1kZuvZxRds1Y1eV4ghk1i13QpXn0C0uzagcPTucUr70aqces85kQhUal42k/SQLTkknhpEOVmCVdClybgSm6xabcQFaGXgCcKdPgEvw8PDw8PD448AhaakaFM0K9amd9NX6qCZoIVrklkIaTOvQfp2iwjJuq4tkE7B1awEg6DpKFhwYrtWJ7KkEJENtjdK0h70YYpcy4RZbKVitRKArKilayll98b7ogJQML10fTbbEwBaDkTQX15dSJZfn425cztKrJHGbAIZKwWtrwTGVKmO0axk6S0+dA5C6YlVe6fYzmFa1WAEVQJTMBhCGrh24AAlDOZp7U7ViKXDXEnWUpVubIfuZqdFctNTYbRJ66AKViDurw1ivsaYxR1JqBMAy990qi8EzpeGbxgrpTGhpcJFUKK/UXZ86uaJzzevhGlZpEl1NdVANHwC1TvMxCZmYVCJmCzBEXYYiCBGsVydgxIC9pgGA6s3ZuM9Y4QSmi7kSc1HIMmQjfVLLGiHXRUUgGDAIXYT1MneFBAgYrCN7rmLMUeAombPq61IVFtGN1IkHYDo6+hlbdo50adrTG8P+oTLtZ7pNNOrKzEzGehCSQaa6wAYjmybWzyL0Zxq22AE4QWgLLuMcrR16HusMIiZbvSlvuXvNBUM465iqsk62YAcbOGoVUM+B7VXLMMr0DJilR6MuG4k+rxaeKBvOFRD2HRNGoe412nVsjBmT7SVrWKEpYetIK+vRuSmq9zdvYYQKjrqV4LiGq09soqHNpRo2CVC0tY2GLkfdKowqY2oRROqe5SBj1pUgag+ik2uuWXrU1ed+AvRTPSSFatFQNN5wrbTFxtN9aioMNVS0XH5ilW4TEyUmx3u2Ndv+QQjSuXTx/gBRj9ni6rxQggTQlXQq7ZcO2zpr5e2e5dRKjWgxQFm6dk/DwwtwXWEpyHTEnPHnNIXd7NJZVzKOuKquYnJFRVJJI4hUqTObL8bIPrQydhRPtMxs9RV1jAr9VdfUp0mDaBDiBW4/y8fJUaYSyCfcjmRyKHmpHjzzMIq4Inge5FcRNVcWsprrKkN1DOMj6x4u7TXb3DsTUSydgsMLuNeViEbzGR+Y+RgN+68+8BRgZB5XbLSoHK/iR2XCUX1lgyEfc369sO+UgKLpfHgiVf5QWw4TpMpjAkine2gmuW3ZF/6ubudRhuqYtmmEIJTah1v1bwVDQtoBBmu1sJkuwXQEcdhRJf+MX59XGoqXtUokdkNvkPriRM25FaJljqDPUDa01IsJzYaOQhYhblvYjNjklZZuS5KVfIveMIaUHFx3ZYpDgYFOJysWEiBwDdghYZsKyBQYrtVrLlNR5jZWou5K+/GcqQ4tRt+JKgX0zXokc2/cuXjGrY85HGhfYcX9E1bjDeqA0K6AYACklYYJ5uA8YwiGbOTc1WYNRUvlRJvvGjqZQuejzccA1nK/UdWL9B8MyMcZhIllqCiKxNV123ZsQ9csgFUqqNm0OP3b4YRLOeS5otvtnPTdeqF81Pcadpkdl4k8rrbbfLzC078eHh4eHh6rhEPSUh1OS/BU0WlcR8+ZiElDmVBNgA1D+56xVMGAxcUGaFlpyUozM5HBDOmc6y06WMeQrdVNoRcDWxh/YqyHXAQY5MZEdNpt2SRQIi29HCyEI5+FXafFNnYHlmbpPEki2KBU2mIpQdTjI4UbTDN2YoBsMUfrViN+K8gnCOVYaStHyV5kc+xYqebEBEmAKgW2GRD2uU0ah3S0MXHXBornygJLdMPleJGQt7UFGql/NhCKqta/o7lkTOhtrlR6KR31LiNlXVm2IXdBEcQq99kUXq/0aBWhe81KIDCWJsdIzqqymt1nokIpGwtYBuq81loNHTVetNTcmejidLZCVwfMBnVlkxIskpYO5kshosNUeHUUCizMN1H7raJ8k4XR6Naxe+jA6yBggC5mHg6AbIyhu1W3SFsIEWn6sawRZF1ioO9v/ECIZMFYu26tR8uKbjRsRrzErInPBIAItmg8BYSlBXVxP1+sozU2sI0hBDEUujxPwCW6fcUhF/0YcSNHrPOLu70U8gHNrdckeN8VKlE1ffW19V0wX5Arl4JlXFrCPv9Bh0Em5J5zuHrQTDLIiGwrOyaV28fMXZXaByr5p8IFoclYjcnQnfUZd54yhYq+1vdP1JyFXm2OXrQUE2RqEFd7qFbzUkUMgLk1V80PN64JO1cStgGAsb6Nlamax6vPeFF5jgPlkqi6LaJlxx6N5OnG7tyq4A7sb1RlJn1MQ6M/ASzVQ1KokqaKWF4x1EltwKH2rYmaK4SQjylBBwDIOIKZyPlGmmTLtjHJkE0wZOMmIlSCNdQXJ7fOo6+F6FI3RZKUtpTa+NouunW1QchBiHDW+U3LBkHU1R/xfGD9WWrMQHeLTp9YP4TYp56O5h6OMgVk24Tswy7WwXrl87EPROkET9kgyJa5MA4wgObVE5EucNdVJybIWNqOMzKEfR31lN/KbLyAo4uYcOkqImUIlgi1WU3TRQzDSoS1KQ234t7xShoO0xuB/lq8wEc2krwNDNeQ/dt22XG59rqBc2WTCRzdKxIdkVjZrFl1k6nQyBQAoamQE48KXzC3sQQZbFH+8IRFDAYx+C5138o6s3R4WSMUJo2FEaKkRNFR0qC2pYM4VBO5uNgA9UMXfRm6VJmoo5UmO3mo0PkMTKrBD9YydI8orYBp7mK2Z+3yVoZiWiCYV5POS1dYXaQYWae8dApmMHRzI2P1j1mXAEOgGzoQB5YLjn5dTRDjErEuvC8lQ6GjfVkWIBcJ8kI9J9FyAKafu3ApsGktgEp5YdpZL1JaURnNdo4JJbiOEhYJqQInRjcsXPpUOFT3r1qpaiSewBTN0OvRCE4mnXKlGpEzm3pUXcdMAsmiKZsJyB4q64CNRMGHfZeuRqziu8yd60ZGuv+p6buaYqT5Ny+dcCsazv+v4ijYSOS/eXaraTkmEj3Sfv1q4wFe6TVtxmgjvQVGFDw1WP1/iCeEQAUOUaHKBFPCZMhsZRGzMIyfMJ8SluDn9RLo6VyxpUCVMtQL0XRKUa/VhmkDW+oCR23aDwCY79fQXVKrkgWEjBjWrVUOhcVBCtI7ee3eGP8/e2+ybFmSXYctb05z+9dFH5lZ2VSTAAoASTUgJTOSRplM4pAz/YD+RKahBjKZ6RdkpgmHlCbCRDKYhEYiCwVUk5mRGZHx4vW3O603GuztzU1Ma4BAXDcD6kW+9847148f973XXnstM/MYL2hlltcaOhg6W84qWMGpfWmjm4jbFiiCSkoobcUaENAHNZ7ZoZRa+SAOVFPcmjfud0HSLW0CuEiHnrBpA9XtIbmn2KRNWY6IGWj7VMQAxAuPxTep3rJ7KeNGIgf6vUhqkInwE+8FQPnjDZqHCYp39GyKbSIB2QrAKgUM8NkmlpGRvPKAyBxaspYC4VmejzPGcpuswQT/v7Ch5pJ3eS3aSz7Y+Pd2nxs8/5T8BR9N9njXzHHNxKL+BBg/YRWsTpEDEkhKcVxXKE+7OAf3b+lkK+416msRrfaKvYeZpmx9KDICm0vPwkyB3aesTnTSQxqJ6Ve0Pie3DuvPaEK6ZwYqy1xd6TEGVMIjKVZ5AW+RDrfsNDe1j4EsQP3csaav6MA0nFlWM4uhp/U+risUD5murpAx27V1qv2hcNC7VNs0k9Q76rRPKkmCJC2DQ5McVXxmtvaRdAZ+fOH6w9IREbBJRJ0c2ThY+w+Zvq3ONcMBvUPMQkcF6JaDgh4p2NFMluL70o2PqkzhnTgg0rmM5JbXPIWPiEjetqIbOujDIUu12PChiTQn496Q7qt7hFh/thOSHgx7purT51Qdzwn/njSH73F4z7wC9OjhyrRW4f9h9Kkea6rHcRzHcRzHcRy/o/FBZqrVlYSq5EGzt5mRmLlZBfortc4AlAWG6FYNYKNy/rF1YrAFc96QjcmpwfcPlFEo5TCZU2g3jgplafHdDfW1jPd1dPSwFWWp6iFlpyHDqh84YwvRnAX8A2WWeiejt2WoSUZIk43VAWByLaB3odkfqDYWpua68jzNR/1A6jPDLLALPUquMZmRsp886iw2qa5pJiK2thDLlG+3TjU43VLWNC5UvOcY0WpGnwNk/cxSGxIAuVP4+A/eAgBOqwZ/tf8oikHkQhNeUaakMvPkEFl7jSiuHyA9n9V24n0UgcXKn2XIWJSeGdHZG5S3/eTZLjzQX3DWuRqg+Afvuinut1P4x5ydFhb+niZLb9SBsEfxyR4fX5Dszm9fP4JsaN7qa4HJtUfR0PVNnbLuIH4f1kF3llSZmmcOck4pyrgvMP/bEtUD3df2Y4n2KWvxNhLT72ViiGugYKa3z7Rb9V4crLmcSQ5JcxJKBHJMa9qcj4BH1GIuS4OOlZHUXqbSykiogw9C7Q7RY1fvJdXaQ3uMAUxmrh1F+OcG3kiom4DdAiOL/vvaorgpUN2xApJKtVFAxPo6QJ8xT6jCWtJDQgHof9N6l0ageshaZyrELG0o6J0BDmv0YQRGOzw51thYt04MblcC9V1Q0qIabRD6mO6zemtN6NN4xms6c7NxlUf5IA8MAgxzTXT+LBjpi0IrRaq3ygGASB0KXv1dgQeA5jIgKmFOTY1onPE+jw/yUPWKi+3TVI+LG37A9QXgWKy6vE8w0bjwZCrcynitULfTjUD71KWNxUiczqh48Xy+xmagHW03VLi6X8B/S2/f/Dq5jHTnHuU7HSFlr9NGLhz1LkYpu3V684qdiDZqkbgTDpseB9R4V2Vw5AuF5mlqQ5m+ZahWC/SnQMebqy8d9D239nQARAatuvRy92z8HpWSPOJpI3xydZE9AAn0ywTXxvrTQPW39hN6A/XEwPT0oU++WOOjOR0u32zOobTLNoFEAvLSH9DzZZ/m2AGQUdcPhxKJSAei4fURelhzuURXpq8B3mizulX43rDysLWLxuSlcvjumwv6nVGivlTQAZ70wJwDl3HuMZzTQzr56AHLusftfhpvMJBo5q/ZiCFs1kJk0CEdglf/lA/I1QA3sMWdcpBvaD0u3lAtt2GDhOHModiyatId9eO68KczswQxpvYmgMh+sZfRZpC9A2R3uAmbOUt0dgqykag+o/ekUBbbNfWdiCo5EKmGnlEwAIcXEZKWgzio19lawLELka8cRLBp7BX0vY5wsJu4eP3pVyW8Sj3hXiSikmT1n8BDUG1SEDITxGAz/DuWf7I1MrkCqL+b77FKtXWnUluVV0wmCxByITByYDsuaS7zlhaZ1S9NlAAUB+WS3IlKDgS5BiUw4dJz0XvaW0LfKn1WDjK0hwr3WNBFQyCQt+wI/4MA+Qe0iCjzKTOyEmifG048XOfxvo8j/Hscx3Ecx3Ecx/E7Gh9kpuoKD8GwRyKsALZ2McwQWx0b3l2JZJxcOspSObyz0xSVNZ8P1ITOhIfZqoPh1ph1P0HLmJQHYK2EYPhqXCQD5KhqFAr9GWHHzIiRGCJVM80gXpn5G14YwCQrqOVvkaLdM8riHItS2NJF8kfxIJPQtqKMV2+DfquMkWre/A8QOzJnMuaEhJw1KTPvUN1xZseBqbRImJoXGH7S4mRBk/LwZonZU6I8PltscddTuG+9gGk0dAYpi+wa4X7C/LjsWUc91ZIgsJCF5IQaYQSKtYwMy/weJV8nXLM/O/TEjOLmjYTsJZzkV+1NiWnwYOX5CNmdF0jm4xUweRoFWnG9nUU7M3ldYv4t/feicRjmMsHXEih3dP/NE4nNzwfUC1owSjk098zmfp0Ut+wE6B4nQHNyKSOsrXqCHx2LRowCUTBAKMAgoRxeZWYHYyL2uMIflEyEIyZ7+Ho4dREuvLtZRKNzL4Dqhi35RtK3DSx8tUvokdek4hNbWUofoWExMfAskKBvNSFQ8wQyVm91nDcqAXE2OojU7qEIPdEZqze0jPlMF9wzzJ3WT0KIiIme2qcEsvkwKbsVntqpXAZfR1h7+IHmdkaAk6PP5tcT+5f/rUbAhPuVQHWb3klbJnUlIehZ5d68MYvdyfR3x0Pd7mKLaO/oNSFY4b0Inws4zGDD/QddbDPlzoCjotL7OYQTZEg8ApZfgGIroFoVF5GtfTw07MJCstuE3Eu4izGixLZRQObx6JzElKXV2qbCOCZMsB+Z1WgVyspgELTBqR4JsutwoF6S1+0US/kFuBYisXghspe0l1CNxOIr+vewEOjPExtXtQkiFAao7lPfW8Eb8jgjiDH07QUIFKDD3EyTylGxTz6adEill1EOiC0eqvPoLgILmeEfPlDGiYNlU+4XH9/COIl3b6nmvHi+xaM5HarOC+wGmrfr+wVE4Q4E8MNBkW9GcR55w1dd+vwk3J/3cibWqhwEb5L87yx4GGcEBwYfUwBQax3nOAZrmuYpQPWqS/Jx/TS0oaR7DtJ45vGAH53QxC2KDl/fn6Pl9qaL/49q4QBgJnSgDosExe2f0d/a/mzEZNlFf9LdqxXqG37WPnl2uoL+bmB9mhlQsmvKsKAWiuD36yoX50eMIvbOAlzLC3M1pjq+02AfVj5ktYfki3hHgUf7LWGh4nSIRt7FrU7vwcTDLNKObCcuepSClYBiUCYAzzVatDq+u670cHMLwVwJ/aaKcz8uiE0cIGX6hfRZggEHQF6xUcIwM80QLIsZXF+8yMzkw2Ea5kAmIX5XpNaY7oL/Jt9XsQNkm8oDrhQH9dzQB+tVege9+AFbvqCgBACqjaAe0+A0kw1iQCfvVdVn8K0AXAhe9+LQaN4lSU2nOLAOLWUCGIPMan/Ixg9BCN0/K1yp9x/+/SAP1WD9BpeILCF7iZvmmcH8EW3ku9spVJd2aHlZxswVyxGCF4JzEkVhYQxniIsGs2Cc3NbYbdnGykqU0yFpBHeZlJegmmd4GVWbS5b5g4NU+CSrSFZV9HOTNwrlJkWB/VkexQqMy0yCzSZiRbFNm/P+Ywe3MJh8xUSoDmgf8fUeGYiZweRvmVSzT4clQBtIIKV4dtYAqJ2gfZbaMaobGZv07dLi5Mk2fAu3D3M8eUbKELU2mGiax9YUuLwj8pd9O0F9n7Jr4ZCyF3HYQwjhY9YApNrOuHCciaRnESJ3r/2BQ07zJBkz+4LaRMrrUHfPDsuTtDFUtxLweb0va6KfAGqfrt9fOFQvKTt9Omsx0/SDb/dLPNzMsfwqVGs8ti85QFtQIBHaM4YTge2PWV+5cDBGorukzD4PkszSplojAN8qgOvW48yn2uLpCKEc3D5kjCLOVd73GjffKLNIjkrhv4lOxfYPasWJjwXDwsHP6V6k8vBDVv/noNfM2LQ9Y4A5ro36qYVo0vtJdVIZ/1bIHIPmtrwKagSZqIun9yrVfdOzVvcCriRdW4ACx7xtJhKVOpKWDIFdf5JxCxy1oIVM1RfpvSZbPL52RWsw7Eu68ykAtpSRDivmL5yndiG9F9FGcFg5CHvoABUEHsYl4jsX5kqldmh6BqHOKRNCB5kO2yBuEqVAs/Yp4XHgUiMtDhCu2LOatZ2F36tuZLRkfJ/H+/8JjuM4juM4juM4/p6MDzJTdYrrQasEUZkpwVOBlQgnsHtgkfKbAmbFbELlofZZv0evYuS3Ot2hUC6KgzsvcMeMzebtPIp8o/AY3s4AZjLaSiVfxyXVn4IiEUFn9D0yVU8RqK2z743JDUI3DHcGZ5o2RcjFBjBPEGHLRy8esNlTxrl5qKNoO0oHfV1E9ZYcoirvFHCrYtuF+aMWI0s6YluQjyZfptil+spwmp7B/BW1aQTvz7Nna5ywVN27zQLnJzsInuNmLHCzo2xrfzWDXnMbzvmIrlCYvub6nE81nKBYE6N1m2pM5HST4DiPlA3Qf+T5nVHE32bSdZH5agRUl9oP2icedmbj3wr3pPpDRmiOIOi9YGELzoieNjidU5qzaWu8e0fwt28UVr/UkTl6/yXX/wEUW4liCzz8hK//6QYnBast3c5h3k1jZjlcHDYsyLAevYBvFSxDvLJNEpcexGKPbSm1h+MHKs56TLM2sf6hjq0+0gCOZS391MLXNsLjQIInXcU8Bjal8NJD7nju2pQ5BkGTAKN6haRy1kmConVg9VrAZdl0qPM2iiB9/pbTh5lTYPkCh8pCZsq1x0w8JKAL5YM4cIPi6Yz/joYDU3qvS1YJE7sMFcs8jQnVENFLlzxaww3TvEXjDO0P3IpCPb66lXBVut/h3Kb0KZSZuEUw8EZo7jnjzJZ7qCPDJ+Nx1VEdOdawKxzuSVkpymTvjtykjFaxAUU+32oAkJmWv6/jgzxU22ek/Ss8oksNSoaGwgvdCYx8aMhP93BcG/WNhmoESoYphlOHs5/dxmtv2wodu2d4IyFY3g1VIk/IBw3dJNr7uErwLEBQYg7J6oYh0kqQjmyAnlzakG3taTMB11RbifkrWsHVxsFyq8nDlx72aY+zU4IZR6NgTfCnSi938a7A9G3m2iEO6fvDIxu1jMe2AHa0lCZvqC6dy7iJcPkOqN9xfYut3cqfUd2wHwvcsObrybSFB3D9QKePvZxGCElrwD3nt3tfYPpapb+V1XKd9lEiECCHn0PT6QChE+kiWbUlWFG2EsVGxsCLfp7/15LzT6gX2YsBgn+u/KY+OMB9FhiN08MgyUsAP6bI5WzRYN1QILe/mqG8oR+s7gWGE6B9YeJzqr9jFakdsP3MQX+05/mxaHuGapUDTgcUE3pOBYBuS9CnLCwmE9rB9psafmqhuG/V+hJqSV9LAGU1or0OclcePrSrtBp7Pryq6QgYEd2K8p5kUVmoqQvnJkQrgYDWzkfAi3jAu06h4vqzzHq0fyhvp/r0jqByVO8Lz6aXMZDwMtXt4Qi2D84ucjxAkw9k/4RPrSWhtSpq2orUSlSuARlIOpJqmyMHP8OJpz5cAOqBiI8lB779CdC+pOep1+qgrSXvP90+97E9SLUCZuai21XQ4gbo3oP1oCuoLh6cjIQXAJdjoKlsEYJKO8EBnJ+/Iz6THMwlOV3BRKowHT+oK9s6aYH70iezepfmF/5QkxiCgu+8Le19HUf49ziO4ziO4ziO43c0PshM1RfkbSp6QeQHAGKtUN0nKnv7wkQIaew0fGiaHyTGpcfAhstqZnDHGZUzAlgX0UViuLDwp8lMU7CGb2iydjHLNIloIQEzSky/T5G24XYYWwHwqXXGTj1cgBxLi3pKf2voNEylgVf0YcapwMOXDHc+6aCFx/0rwmL1LkG1svIxMnUlESLyyD1kC+VaQFgN8T2zXQ2gg8i3oig5QmxFEu92iqJ3AFj+3i3OqyHCuqU2qBi23HQVdt8uE5FBJLgaXqD+7SR8if1nIyrWxO33JarvmFjVkCGCyczTI1PaZ6xDwaYIDCWKUcTIWjXkXBIEzeFStl7dktH7yApcUjvob1jsfZe1HdUsNh/UkeoEBWsI9D/q4Rt6Tu+aFcSGvq7WIhJD4IH+1CUt4Hsdoc7+lEhei4qefT9qDKxIJLVHURqMAz0nN0oUU7qRp6dbXG9o3crrEtWtxHDCz/N5h5qz2GZXod1WkIxKuFHGrFIVNrY9tUPBEG2GHYY1sCsgtgo1iyTYicd4yvOmPDmlBKGVdzoyX53OWtkUXS/oTXuFVKpw9NyiDrFNqIT0CTJ2NaWjwXVIjgI5lVYNidHaPvaRIVuuyekmeoaadP3mWYZiOAFXZmtO+6iMVl9Ta5bJBFqCtzD9Lt9DS4zr8J5Ik9Cd7hG3v4lAanKRQClcyuq7x57QldCqVTogtHtpD7HVMVsXJhGm4Bk9CbeVuS15BYzB6aamzD10BOg205uO7T/8vawlTeXQLsPMcW9g7V+feQ+/r+ODPFSLe4JHvUwbXLEVzBblw2cUkbLvrUCxoBVRXYywVqK9pbfDtgrVknaBwRbwEwf/mFfKXQ19xVCwSi0otgbs6QjN7EvTauCEVpNvNMoHccjc/aiN9+E3Zdw8XOUiJJPbZ323Psf5/6OhW3pbdi9lZF7aRsNvNIpsw3aB4ecR62ooHQZo1NccdIxITE9J0M2BZB9vFsOC1GFkaEcw6YDtHjv86OffAwAmesSmr7Gapk358pJqiGJTRMeMcH3R8uZ0JbD9Cc3bk4/vMBiFtmcnk60+EEWXg4gLPN88zDSpLTnFdfW8dSCYWJ/wZ+lT/TW2LWhuQeDNSb2ukySizzdPEOyo0t8LB+5w4qAuy7SpicxuzKZ+ZMse9jnDdTgLsl0Erz5c04JRaw3PrSfFaYf5pIer6UIvlpv4+6/uT9HdURAw4f5k94I+nNJJ1F5pB1E6eJdqnsH8YT7t0fDce2YCh/lXTYISfaOID8DrbFxZoGLewSiBUUa1rgjVAgdtUV7RgSgzeFJxH7YrPSB93MiLbar32Ul6RohlAn74jcgclHCgjuYFMLni4Kqn4EjmcpgRwk+1V1exyH9QEBsEygz6tHVWFsmtDZE4FP0FGaeHeVCtiHuUGgS8FrFPWLYysuzNNNlX2qmLpSb6YyLuE+peo7xPXICDUk3pYWR673Rmjk4G5nyNjpyQwjXGWer3j33obVrHEcL3gM6YxXnvcvj6KFP4ng4zp7pZXugPNk2hn9MvTGxIn580eLmigkUzlvj2V0/Si7oaYcbQlAVAeYy8WdVXOh7StkZc6L7wgJEwTO7REwPDkbraKozz9OLbZz1WM9rsBqPQjGmnkZ3E/FO6rxerNX799jFd767A/rlA+ADCpj5JvSsPahv9uYcN/X9WJPlFK4ikwy0weiuihGE5eIwLERvgu8cW4oTdcoyEuS/ihgQA7VO6xtMvr9CMRfzvndHYd7RjNNephhhJIqFOY1O9aPMnLZ5e0OGw6yo4JyD5BPeTpBVwUek0AAAgAElEQVQoRyaiZAddaJ/wKiOVSNI1FVzrkmPagOAJDQgbV9RlBTXzq1agvqa/ZybJd9XMcFBv0oOIh6CrPCEaoL493abevfwgFTZdQ/VM2onBlI9Zmp8b2sPZxtAuLaZndKPOSdxvprFv+nK3iHaDpbaozzp+DhJeOzziOnupLNYtrWEpPM5mDd6tqY9UFwKTigPMwkQy2d31kj970J/NvGx7AaezDE4gWsnZqaMsM+sJPcicov4xZYcya7ex0+x5GpG8ejOekrBJcAGC3vdQD5VjVoMvDi3SCI1J35MmEXXy/sruIvXPCvN3iT7hM5PwgYjZriuSBKNuRCIfzQzkTqO6yshJ0QuVAgvN73KxFwc2a3G9tJIDEh/vK6AeRSapGH8tkxS0dQr+VZ8hVTJ9/jDnPjs9cunEPGvONZNVlwLsEDDlYjByJH3t930ca6rHcRzHcRzHcRy/o/FBZqquphpDeacOKOWGfUIBoLgsY71Mnzm8XlNatr2dkQpdQN8GCRvqEzuN+i4ZZXuFg6g4yJmZM6rXBhccOygIro3Y0xHusSM2JYCTaQfDhssj+z+qvYzXOWX49G9fP4HfUNZXdpTpRlm+wkch+1wMHGDG4zZliOMJi6/vg2EAt+LsJKo1Z1vs+RijZA+IS8pspnfiQERj96nF6Sckz+O8wIs5pZyvtye4/vY0ZsbFkOj2QSwhsC/tzKH7km767HSPLWdRQ68PaoZyqyLECIgDQQyTmSdAJi9OOYqDiDxvYSIkI7VMeJ1kBKuHdJ80P0kgHSJjUXpqKwhsS7j0vfpWHLAdnU5QXC6lGCT4Qh01b5qXG30gegEA/YZupLqRKDWwe0ypztlP32FeUkrR2/TqDyuNJ+cb/LPHXwMAzos9LtmJ/E9ff4F2LPDZBTHcH7pJ/FN321lUDCsmI0ZfwDFrVQyIaYnqOSvhz6026u98Fr1L78nByOQXickbslNxmPluE5Rri5QhmqWLa1iv1YEBgFcMW/Mcu8Kjvk3fDwIhnl1YwpoOcDAAmIWNTGlnBHynIPfM2r5LrUnByUX7gDYkcwkvyIMYAOxQYPIuvQtep7YUrzyKTX5ND8V7z7j0CckoyYg91pizLoHwHGKrTFZDFZ7g7IgGZBknwM+U/7tXaX4UO9OE6+dM4VzwIQjbALTObZ0g5qDelC3j93Z8kIeq3ioyKEbq+XKaoLSAG8lFjzOGXY1VGHjjFoWDHyWwCDuehA76pJZqOAGu0ftUD+kf22i1hVFCrgugox+c/2iNl5/QYbPpa1yv5+ge+OB4PUtqPBMHvVVRSs1OJV59x7pmRqC8y6TwFKADt0eLg5adA/m0ysU62HgyQjAZxjqN+kqjvqV5ml76WAtsL4jgEZxL8hfdzD2KrUDzCf1wddbi9y8uAQAv6wf81cNLAMC7V2fUK5e9RfEeJdVfixfEfnq83JNqC4D77TQqVj063aKQDu8e6BDx5wNGhtHloA8Mne0kwVpyPHTYccoDdYLbImwZoLLsEAsw9HBK5tvxWdQ+HojSJBswpwGzcPAiQXEBivciwWVhRFJNwf2LPL+qEZAqkNfSZzErB89EHYAIMQGWM1PSxA0H06tvHkHN6LlcnG6xrGl9L592+OnqCk8r+nD34wz/7rdf0jVGBT9HtAc8mzS4a+nB9/sk7Gp7eTBPwiXdWzPxUdYPoOAg/rtw5Nid6TeHerQrfRZkeJYBzPpZIyxPVoThoMsVfeAA1Sq+tjiA1Q9qttKjukuHmZmlXuZgX5YgThHrlwBgmYAoWoVymxyEXHa4F1sKZkMcWuxFUtZSSR5RWvoc+WEWezl7SfAx32OxThrNHoCdM4Rcc19qaAMUiPreGAVUlwUgZTYfgoLuXPYzV3qLKmyeApccOo+KWvw7sQ9ZZKS9WdamdGag1okwZaZU+7YZxPy+jiP8exzHcRzHcRzH8TsaH2SmGoSvnQbsKYVRs5MW1kqU3NaxmnSRrLG5mkdVFnWvSVCfdVLLdzq2agxnDl4iEhC6JxY++EZqFw3F5SjgC4+Pf/IOADAve2gOA9tRY7ycRo9G4UhHFQD0TkE4ui5Af8ePKS4K7DxnEsEDoCwqF+/OiRayl7BBRWqQUWy8uFeor9I1+pVAf8qZkvZQbYrWc2cO2QmMK4eS21z+m5/+OWpO/f705sf41V98TJ+FTaUjC9Ek4oOtPNzE4WRO2NBgNHYtZfVPT7Z4OiOi0h8svsep3mP3nJ7T//buS3z9N88AAOPSUUTOzyaYTwM49M39wchVdSz7zkbykKE2GgDozy2KjUxN7tlncULEDGI4c1DbQ3JMgLzMlL6ObOBVWju+TM8sZjuc+dkKsSVFOIHqRqG642vOgDHXax0AxSIB4wlgFYubSAfJqUY7Frjq5rhh958///pjOF7famrgnIyM34dugrfXVArRlUFV07M1RqHbVhCccU0u5QFZxU5dIvcNIgpICE0vYn2TGOiJYCeirnZgoQc9XiBBmGpI7wjALNK4Nj0Ew/e6ofciiEa4wscUq1wTWao/TZByJBLt6b6C0IfTLjLmxZCera0oi0zPM/mkhv8WxB/yTFSOiEbkXgHSJtH8Yelj24+QzIXMmLURUu8F7Cz8kiT0KRf656xeeBwYdgiX2pbUIA5gbpM5zMjhkIz0QwUymZWCcgJSUE4CGKL+hB6G6KlsEcVl/qFgv/hAD9Xx0Qg5Ubh4skHPUOIwaJzMW2hmkt43E+zfsHOGAAruFRtXDt4I1N/TyisfkmuMPRtpQQdlI+nhu6TEFJnGnlxZ+qAgJBx+c0swbvvNgqDDsP+cJiPlYk3swfBvN7OQNa1mZyRG3vDloA6UUuQgMLK6imoEJBIF3i4z+qATqH5F2FuxA7pHiL9npy7e0/SVpnrIPtRwgPGcP/PZiE8f3+HfPP9LAMCL4h5/29FB96u3jyO87mrqiS14TrwGTLDd+tEOP724w7stzf9q0uE/e/YVAOC/Wv17dBw9TEWPOzvHG7bq6I1G8YhOrNmkx/27JSRbezkhsg0o66ezid4PpIMU4I1CpLarcUZmAgAgRnkQQFEd2cevx1XS+VNDgo0VM6fD9XWTMSUz+btiJ9CzSblw1E8cxfAXNjE9d9QesfsRs7T3IkrhhfadKBovgekJzc9oFS45aCyUxa+6R9htuP93rxFkCaV0OJs12I00Ce9uV5gvmI0+arQNBTtCeOirIraG2Dqray7cwSHltY89t1gXqK9kPOiESzBusUM89GxNh1ZulxZg+uGE2lByNmro0SwfZKzX6oZY28Npapupb+jHXUkHczgcVNZuA8FShUFOshOwHHyqTsX/LkeBci2ioYLep/e4feQBSWppAFA9+APYP6qkcT0+MtULH98Zrz312AfZydN0AS9TYCsGeVCb9pmqqt4Kkg4t0vfCwf9D20ZXpjYl3WSQuqDrhTNQmkMGMURqtxunKQmQ5wO04mBnlFSbzmvfpYNr3/9G1SP8exzHcRzHcRzH8TsaH2SmevFkAzXtsWsrtBsKi8vZgH1fRkhsdzUD2CxZ7nTSinUCxdsiskX3L12EkIUA5DxlflJZyKB+NCi4iolErYAYBC6/pwzrav0oCgfIrOAPEEklwD++OIR89J2GK1lRpfBQu6C7CiLcRPGAhKuYhYerbYSzYUWCtjdJ2L997DGuLATbd03mPYavKXM0c3/gmWpnDsWc8OafPLvCf/viTzHjP269xJSx6LoeMbxMPYPjvkTLWdsXX1xiVVIW1ZgSbzdLjJzJ13rE36zJVf3fffVl7Es1RuHnz7/HqqCH8XS2wd2OUp7NdgpIHyN+vQcUZy+2TPPrHffnMXEm1wj2gghOgdSF1QgEayrvD9AAM3MoGGKWWd+lagWqew81cEQ+SWxi3XgMCxGJOcU2gw4XGRJQUBYcIP3qWkWSjrRk1RZ+ttgkFIJg0aQEtHq5Rs1WhE+mu4iO7NsKdTWiZCi3b3TMjj55dA/jJNZsDPHkfB01mc3NJN4HLJuSh3/WPmrP+soBRiTVKitin2p5JwGZmNNO+UTue5RQmiCiENjdshdoLljVbDnCdioSBnP2qRrSfJsprd3kVZruNxhoB8KNGlIv97gAukdJlN7MPcqrpI4WMlrVkrVaFAERQPs5fVMoD2wKWCZNDSsRCY31LSIN2QtWycoEWaJ1XeWgtgpmmSQSBPcnSwN4G9J4Qn7ifBuB6krFz+V1RiQqfSyRBFvCYKLhu0RakgbR0CHogEfPVI+kTMW30J3RPY9LDx9U34TH0LJm9Y0mlSZWSpueN+j7AkK8/5nqB3mo3nx7AjmpIZdjFBtXimpM24eAQ6Wfr26T6339TkJahnMAuIsRiqGgsNlbZt05KFhe9MVlGWnzwwlBQXKdyRaGGosALdBISRdR0N1L+r0cRnOTJK8XFrSdOcguCfarLm0e48JAjDJJmEkfRSlc7dAH6G0joXcKloOEdltB8EvqC/F3nEAen1KxqFYj/ufX/wKfzakF46vdOZYl7U7/6fNX6N3hkvu9+VsAwO04w7ue2jh+vvwe00c9/u3rPwIArPsa04JbjOYNrm7o5/ymxJ9vfxQP9I8uHvCMjb3vmwkexnlyLinSPBKkS19XN5IUi/izFLsUtIxzrtWtWO2qVbHVCJ5g1lDLq64VNENe3aME7ZVr+numzg9I+l5/Ig6ELaTxaJ6GNeIgntCGLL+rD55hLgTvBanxxA1OJbGAceVgz0b8D//5/wKA4PKHQNkG8L/XfwAA+NOvP4cUHprlB3vhMX9CO+u6q1EqizmrMjkvUPJ6X3xyj5IN0C+/P4VXGuE8cQsLPeN58wJ2U2RtIh4qMKAVBWd5y1E0bc9Y68Opg5va2K5ilikwtI2GZL4BzU96eXWTBX81fa8IwlLikGkMHHqG5kpA+TW9OmwBimYJC6A7T+9k+bhBEZj1uxKobXRqcpVEdROu4bPWL5LXzEVLgumHGMiJJ0oMjiLjJAggerJ6wAgU94n1nDN6SeiCoWKbpFkjGzk6xyDuQ7ZOsLxuGX7nS9oSEMERa09tM3E9Fg7FjC7snQSYVyIsrc/FY3ppKm2pPUtmmPh7Oj7IQ7VYK8heYawdygUdGpNqQD8W8PwSiNLFPtL2mcT0O6bNe2D3xQjJ2r/T2RAP074vMPbZlDofjZ+Lbeqrs3MHsRyAW9r9fNbzKTlzCr1p5b1I2QD/bFCtcXMDEfr9jE79eFvSKk19dh6GX0zZKDJjDtykMwsY3uAmNvbLuoKyN/2Ga2YmZTz+bGQOP93X9LSN5KHP5zf4R9NX+GX3HADw9HyNkXedShisWc/wTXuCbzZn+HZLu8ztbhrdfX65eoz720Ws63389A5frojU1doCinfJ2fMBb7eLWNd7t1lgz3Z9s5MWsrSwjA6IvUxBR+lRbhLxxavUJ+jKFDAFaUl5GxhgIpKMhKXNLx7aGmiep+CkugnPnWqooe7uC59tyOKwVelRUl7Sz5pkp1d4jE8tZB9UsZJ7kGoFhhOP8g/J0P1fffRr/PHsWwDA6+EM/9H0a1waIhb9T2/+JX66JPbZf3HyC3w6oYLiH//ht/iz9af45e1T+nNPt1Hy8vlsjc5qfPNwRn9Pejxa0EbovcD1dsbzq+hezjjArA1cqAUaCVQO6ML9q0S+sYctMNKmA6zYCgyc7bqlgaotbFjvg4yOUpIPmxBomKk/kHsMB6cwlE1GQlyREX1awdaK/L06q3MqCjIDMU/nqkQyEZ+65yOKZY+LJdv3NXVsO5K1hWs1NGuNl/cC1QPvB5WA4Uy9e+Rhpy450KhMZhG0fmJgh5TFRrctAGormRCXkaQyKUxSiAp1d08uNkjPIGakMiEg5MyDeD1XpZ/Ps/q8dzv8nuE9USgf7frsoOAnFiOr0bVNRepcPpvb93Qca6rHcRzHcRzHcRy/o/FBZqqB/SukR88OIX1TANsitl4IK6LjRpllivufDJgsu2gm3Q4F9tzuMWwqbgngHy5SxNldOLiQLc5H4LqKbQLwKerzGhhPXAzYxqWLXqJeA+OTMWbQZWFjNuO1p2yMLof+wsYI10+ybBQeYlTRkzG49ACAqwQK9sM0C4pSJ6xBaibAeJqYe6KV+Pz3SRz/90/eYs6Y3Zne47/76/8az1m8fd3XWO8pVejb1EfgR0l1piCKcF/GrGH7uob2KXu/ms7xbUUZ7Vz3se79+mGFZj2J1zC9irD2/m4CGBnrzC5TKKruZWwjCFF1YNq6wieYUhHcHrIX2Um4MhW9x6UlKB2AO3FAqKf3koQEQNF++8hHlrXaqTinuqUsNUC+zQsLz+bj9rspKr73/txSXYzXZn/qoF/Q+vvx03dYDzU6Q+vgdXOC35vSc/mieoc/23+O//WrP6Z78QLbntbqf7h/FnWY/+TJN/h4co9fgjLVuhzxXz79JQDgrzYv8Ze/+YS8WQFMFz1mJcF5vVVorimtUb3EuHQx68FtFQUHfOEgWnXQVhThTWatRhjTJj1bW3q4Fc9bbUl8n8spaicjVOsLkJ9qEda0SGYPzEYFAsyc4FqfaY847aOWNcA+o5kymi1TdqfbJOzfnXm4l5SmVaVBUVhy7AHQvZtBsNiGlJ6SxSg4n+DlcUHGGQAxu4UVcb0UywFlRV83uwrqsjpg9oZWGXiZtH4dDpx6yNw9m9MsQ4cAwO08RUfKYmEe5JigYXhABSWnoKQUasldKnc4RWUn/wlBOotpj579fYemiHuSqwmSC9rRJ8sG3ahhXS5m/H6OD/JQDUMoF4v7vlWYvlFRws9WCTYal47UlgDUix7WSlzdUV1PKodhx3WCwsG3KrbUAIh1PNVLyAeGBF9reJ3URfLeyNDzGfs+WxXhMKs80Esotu8a9mUizpQOhm4Jk+81VK/RP6O3TzQqwip+ZmBXPhEcdulQrbbp6/KODJXDBjSc+HgAm5nBz37+BjUryG/HGpcd/fE/+8WfQD9ofO1WPHc21itlKxOkVnjIlw2eci128nLEJbfQbK7nZJnFm+TYlHjo6GHI2uOPz18DAC5nS/xF9zHV6wCUZ10MMuR1SRD2PGFR8l7Fvx17Uaf+QC0GXsBXiamk9jJBayJZvUFRq0PcTEofn0X9toj/ffspYB6NKLkFq74WUH0ioYwzkQ7tQaAIZJshGRGI0wHquxodk17++c9+heuOyEK/ePMMj882eDSlGuir9Rn+x5t/QZf3goKL0Os5H9Az3NaPGjMWxv+Lm4/w05OrqLD0bLrB1y2RmO77KXRtsOCe4Xk14PUN9W7ZTZkMGJRnWzL+aFMTSx9qrQ/cZeBTrdRVpIwkMjnCwAXwBaCvC55fDVRJRSmHRGUvoDOFonLtIyQbTbBB5QunE6wbrQABSCEwLEw6ZRsJr34ARWaw5v4F/+N0iAFr39bopYfccqBbusi3ENIBHlEW1UuCfX94j9BUDw0Y4mzaRzUx3ykKSAJ0qxOpi8JlvkTHimGBNFZl7V6GD8sQVHcCJamI0poVQH3NV9Q4aHVSWX+16rM2Gpk9T03kJHtPH2q7KWMZB06Q2hMAWVmUpYHik/p+PWMLwESqfF/HEf49juM4juM4juP4HY0PMlOVpYWsLNyuiJGp3iiMCw8TXNBqwDIVHBOL6ZKieCEAazQsm5bbfRW9If0gAZUEH0SdOPrFLtk+9WeOmqdDtjRJ9yYcUDwk/VbhUmQtWgEzF7BvKXwUOolo69ukCWymHuPSo7xm4QMFmJBhWQExyiQgUSboxtQU5dLn99h8OkBkjFnPurqzVYebZhZhrt27OTRn4dVAHzm0FZT3aYnlWqju8YAnJzs8Z4F9CY/fvH3ED4hg4ZBlXpztMLLma61HPC0JWn5RPeD/1S9QcGtFURiM1zSZwgj4afLtFMpjZKipvNZxrrqpo9AyENQ8gC4pjNtJEr3IiWFCO/j7Ms6jamRGsAF2HzOMe2Iw/XV5IP4QmdhzgteChZlqU/tHd+5RvyAMc+gLmIUj9SEAv3p4hNsNwa71ZECpLL65J3i8a8uocuScgJ4mwpDWFm2fIPiGbfeGXYmr+wV+9pzIYH91+QKGs6+TeQulLe5vCEW4NwLFDRuKN4l85zWAXsKVIXUXqW2LDbNz27aoccyyv3Fd6EQiI/WjcA2CfHOfztgWpahUUK7D+5VgS698IuxpD72XSdQheweHJSs+ZbZ8UZ/YI9m6AWg+NihWQWbLw/HatIOE3OqYTXsl4DhjQyswuZfpnc9s5sZlastDLyOZBwAebubRR1fvqWyRb9rhb9naJ5F85vuEPcQViQxXbEiD2IfeveznYKgkESze9s+T4UPR0veAZDYe0TUhol+zWRKx0p8GxrKEZsb2uKJ9FwBWywZNV2LHxEIhPaT2EbF4n8cHeaiKmwqiriAFYs3GnHt6eQIsejpgUtHmVJdjVAKxTtCGIzOYgtenno8oq1TwGAcNNLxxrRzsCa/sUUJ2Mm4yXiLS8FUjSRCfNwVbJ4F+V3ioTsZaL5xIhsKNiJuMMAKyA+yP6O1wo4QIkFSQFAzv1CigGfZVXXpR5v/4DtNyxIZF/7f3U+glvU37qxnabhGF4ecZTGwrYDh3GE9409kVsX9Tt2mz06XBs9kG9x0FCN/cnMEzG1p3tKGFA2vbVPjonNitH03u8UVNAv3/x5pE34OhdretomoVBCBmJrK5faeSG49JrTBQoE2oSBtogPrBz1kz09t0OtYWvZGAQPSizT0rt5+b2Ka0/A8lqnsf3VXMVOD+y1RrLO9lhCFln+pUAGC5NGF7RQcUz887uUQV1mZh0BkdlcDOVntsucZvjIQH4iErhY/ev+6hjFAcBOCcxF+/JuUrZwSeP6H5LpTFVbOCYKZtsZZRNSnvd/SeWzyC6L8VEG0Gt2e1TTmk9RKk6kId1dcePTOI9S4FKg5sLB+Q+Mz0XDfUC2ymqW0pHpar9J4WO2JvR6N5RcEL3RQdwGFN576oXgK+8gka1g664HvUFvtt6F3zEI87qu0DQNa6pvrD08JpoL/gOn7pYnsdBLXYhZq8rXAgeemVDzr5UL1A0YY5TaUIr2luXCYzWNynQ3ScpfnRbfqcxLzOWmcy/+Bil9jVwgG2SBC1MB52Ej4nXTv08foCMM9pUU+mQ/Ti7ceCgr0x8UCcd3Dm/T9VP8hD1T/q4acCRZmMyPu2gN+nthQpHaoiiDr4aL82KUf0YwHLWYOdGRRMJDhZNpgWI642VO8amwJFqGvMXVyEaqdQbsSBC0Z46XRDurF53178OUkRfmwG1x6Co8Dcpmk4c8B5D61T07WZ8cZdWrhOAUNga6R6YvfEYfUZFVh+7+IdbroZWt6ET893uH9LddPyWkEagelbzgxOM5PlysNPLGYLepHMZITpKKsycw/H2f/pvINxCpuBDoDTRQOxpLpg05fY/2YVa7jDooikmjftCV4waelJuUFdjRg4wBluqkiGMXNPcntlivqDXmtf2mQ9pj3EKCFDhj5zkCyhJ0qHJ+dr7Diw2AOwW6557hXPeWpbiHKAApj/mn6ueqADdeT5b54JOBbUmL7ScBUw/SfU2tINBYa/zNgyv6F5qwy1e0QpxUfAc+7H3fYVlmWPgS2JrrdztHu6X8lrudvRv3vloqYvVAqs5qcNvBdo+Oe8E2gYhagKgdXpHhtNGcVQFrCf0SFttwWKh+Rd5yYumtX7TQEf5tGKA7PqXArPaw8j0iEbgpzwc7FA5QUdqFFCL9UnVUfPIWakykeHH6gkihJamaLs5DwJVMjTAeKujNf3yKT8JtQf6+oA6cg4p7oykZdRTEc4lwJY3ciEhO0E7CTpOdvKZ+08KXjIa5wAIDQwMoku2hUG6cBdCqQBJFu1wh/ofedBjJeU5UcnpgGxxi8sHeKh/p0TkHyiZcBMqWYbzdMXqaaqO6BaA/s5z/HjMdacu6aMNo1KW1ijIAKSJD0RF01efH8/x/sfFhzHcRzHcRzHcfw9GR9kpjqZDVBTAedE9Ob0A8F5klVg6skQWXdaAEqxobiTkNLFAFGVFrMpZWWzcsCur6L3anlZxNaNUCcESMS+L1RsIlddaiOQhiGxAFvKrOnaAs0zwASHj6zOMy6S96RqBcZtgWHCdcj5EF1GnBF4+uIe91uCXc13s6wu5mMrzF0/Ra0MFGeBN7dzCI443ecthn2B6j7JwlW3DCGPQHcu0bHQv39XQ8UaE6IHq5IO3++W8ZmUysaseFKOaJ53UV2l1g4bFn9vJiXWTEk803v8y49+jf/z8lO6RzmN0X+xFbCVjNG1XiXbHjuk1hvRKOidjAozy6dbdFx3PF00GK1C2zED9aaKtWJyukk1UGEAydnA5GuNYs9w7wQYL2Ri8j5vIRjGPfnnl/iPH32LP339BQBg+PUS9seETZaTEf0NM565Lctzje/xyQ6G63izcsBEj9g0CfINQyqLsrToWO7GdjrW5yCB8ozW7XLSwTqJlksVfpC4f00Z86NP7vFkkcQg5i97XHEacr09jZC3mXqI1QAfWLkTC8Gs7IAeRDgyq2XCCfKbDYiC8ukeRcqohGFXpJAR7bMsTZCsY8jUzCwpEsm9OJDhEzYhP+PSQ/EcWIYhg0qeLxAzbThBCAt/Nv2g4Av6+XGqok+yGRR8p2Jd0ExdzBaHE3r9o68umyQAXHYJO7ED+vMkr+lLD2TJmxhTO4+d+rhuc0EKM/GQRsTMXNg038LS3wvX0HufFKcqwGsBxQxcWyDO9zhN3tDCEWig2nTNXChj/RMbVcjQq1hS81rCTnlf8II4LeGPcynCZ6zu93V8kIdqs60gTU0HEsN5UB5ianByQhBkPxZQvKmXwe0bwKatMQw6tqLqwkZFpdv9FNu3C5R3LKU28xEyEqVLBCYj4GqHkd8+3wDVbf5SJNUkaUQkMoXWD71N0G38uSHV9Oo7gXKt0XxE/+6Vx8U5ta68XDxgO9a4tfN4L8YPn6kAACAASURBVEFNyGuBr27OAQAvTteYFz2ez+mQrbXB5AW9KN/encLtamx+ytDNagCuaEcrthLikz1O5lzPXTSYMIwuhY/EJOMk3uxWuLkPTkAeNpBjTvb4/Rdv46F7v56h5ed0N23x24baPZ6dPOCr3QWuL+kAkFag/Yj7GvdkyRXqklgh1br6BFlCAPKLHQqulW7upxBsOr3+6zmGExd7enEywvJLL9cF9C61XRW9gOQDlmAxJkJdeIwfd3j5hDC915en+L2fk+LRi+kD/u2f/WPUXH+y5xYly2b2d5OkdHMxwDcaYprKEUEXWQqPOzvFpqnjRwr1vtW8g/OISlWiTWbg8nTA4xURoTqjsd5NIilNTw2sTnNUKouSg0rnBc4m9EG35zUMlzpc4eGdiNC5v6rjegwj9lf6OD0wMypniLCp7tKWZGcONkDBlaPDLdRwoSK5bFyy1jC3PhXrJCsqbIKdJQjyDSQpP7EwLav97DSKbVKqgkQkWgkjASvgueRjHo/QHDSi0ckmb2KBReJU2G0Bp/iwHkiVKTrJ9CJq7I4LROUo4ehzR36EFTHAFpYC5thzWruo0ORVgqulESjvkyaxV2m+5QDoLulIQwCGgxHVA3IgPerwvf0LDkBOTQzuio1AsUstTKpLzkvtU0dav21gSSHJgyofrSrtjmVQ+Zl5J0hq8mhSfhzHcRzHcRzHcRxhfJCZalRZ2RSpGbm2qCcD2p7CQKUSxLtuJknH1As4K6GYBDSf9BgYQm5+u4LywPCYWzwWPSTDdO6yji4p7nkH3ybPSq9EJFZ4cRhZmolPLjI7CdUkpRSzyHRBvUi+iIZg1oKdPfBQ4YE1fPsvNeZ1j3HPRJScabwyMEwtXBQdGlPihJ1jTqsGb5tlnBv5pItzMPYajlmw/dJgoh1ubymD0aXF/V3KosbP6J6avsTmforTc24bMRqKGdDOSfzNu8fot6w73KgYre/6Ep2lG/7v/69/DdEozJ7TNRpdRVIEnKLonRnXrk3OKyhchPJmL3foe43hmu63ulGRHdqfeeCij/6hxih0b4k8JHuB/nHC5ap7je4R3//pCMlZzXLZoioM7hv2KnUCv/j1SwDAb77/FLUn9Sua9BH9XeqvKi7YpJ2VugLxaNPUEc6fT9htaRtYIwLzc8okS22waWu4e85UjYA/o/RCaYtLRgmcVbB7HfWsy8kIw8Ln15crrOcTGCa2OSMxZ0/WuhyxKRKcCeVhOUORLsG+npm1AYp3ZUa+07y+GYkQFlFVSvQSisUlrAf8NGlT5y4pTnuYiUfBLHZfpFaTYp8INf0ZtUcF9xYULqIWek9azpGcVDlIhqF9SdB2KIWYXsHehfTOR7EXN0qCM3csRnIvD1jOuT6xyszAc43dccldCJzR5epT0hCEHciUxSblRHR9+jq48EQ95SFry7N0fTNJDG7dJCKk8OSSAwDtx2Pce/R1MkSgbB9on4Q2KBxoYqu7Iv49M8uQHisg2tiHA6d9gsBb+vc/hEz1wzxUtwVgCmAxYrmi1WCdxDgqyHDQeRE3LikdRmafWiuhlENV0qpZ72qMfGgIDeB0wHJJ19zva+B7tpbbC3R82ErPG0aQCPT5CwZavC4xG6dvuX4zD32lqZcu1IAIJuaXfk7XDPCbHKlOAwDbhyn2ZZJwGZcutvMU8wF/+IIk7jpbRDlAgJR1gqrRpBxRahMhyIvlPtb49n2J/XeLxNJsq+RmsbDR7g7SY3naYMnuJ7Xe4XpPB9bd5QpqrSDDL0rAsfHBtBzx3ZYUfWRh8eizh9gvO5n3Eep0E0WbsA70RUR406/LCMONo0JRWIwMrdpKRsWa0GbTcL3S3VTQLAVpP+qAUaL8PrRMJUm9ojaQDCc/vF2iPO3gXrHw/JMe3qW5z2X0sC4A3oBmj/fouZZbLHpM6hGW57htSkxnNG9NX8J7gXLKbTPSR+m3dTPB/m4SGe6uclHmbxjKeECFvsjAYu2/naPm1pLuKQVajg/V6UmL0ymt76v1PJY31Gqgdt9oRSbQPWVIt2JT9cBw36pY53QOcELGfms4QHJ5w2covS89xF7Hg5qMw9NhIFOFBrnTTXeRDjK9FxjObWKEDzJThOLLBubrRqWDf2oAJ6CZ5e+siP24YpAQD1w73lJgG2u7U38gZO91ViN26W8VOxGt0nzpoHYqHVJOJEcizQdu4ANYkYTsgQTp8vVDP6kcfVRv8rU4MDBQLdL6E4BlRSQAFKD9pohzHOUjJwJm4uGC080oktxjQXC+y+BrycHOgaoWCOEO6+BAhP89H0f49ziO4ziO4ziO43c0PsxMdWaAicF00WMIzfBOoK7GmJ2NVuHJksg91km8ZUao1hZFYVP2clUD3HeozxpM6jF5sm41PMOidgGAsyb5fX0gig2fmJFeky5tYO5NrhLDNwhBBOszaUWM8NUA2DI0v3uKGDn6JkUV+l75qoKZOZQcIQ4vBjx6TGSkL05uMLCESq1GDE7juiVYtLc6kowaU+Lr6/OYEemZw5rZuf0vV6i6RPgIKlIARfiShcJPlw1OJm383m0zwx2r9sidSiIVoOg9ZJxvvr5I1lcSuNqU+PjHpAT0eLrF3948pqk31HjvGQ6eLDu0V/xctMfyKT3b3a6GmgwRhn5QM+At92sqD3lVoWSTgf7C4Wf/7GsAwC++e4bJV1Wy85s60isG4K8LjMzU1b2AejOH53lQbxNKYE4pA4qZQunx9OUdzXFf4vEZPZdF2aMzBd6taX6UtnGdtl0BM6oIN1uLaBLhBwW5Uwcat4rhWTPz9PcB6NlIfZkMaep9grb1aoBpdLzHaTWiZdRGiESac11NoiScwbnnXSL9bEp44aN1XU5U8hrw2kWdXb1RMKsgzuuT3O4oSQ2oTzBpyJyEB5CJv9dXIpqeB9g1/A5KBxHIVKOOhCDVESEoZKeuQDKdqCym0x4VlztuNhVBxwC8EVEBSu+pHzdkyXonoo3auAwmGfzeLT3qa84eRSLB6rWC6jPCVDZX4evoe6p9vL7qxUEfMLLeXysFBiba26mHalOGaydAgDJcScIPBb0amL3WkGNgiqWebFeQfnB1zYiCTEITchQQrYhwtldI5gZI2bmZ0j0G9nz1QH27Nsu239fxQR6qk0UPNT1knBalYVUPWgyPFjt8sSRl6V/cPcN8SqtwMBr7TQ3JbFe3MlDsRDGf9uiGAmD2qGolwOeGlx6SITTVi1gXCSMKUtceeiNR36R6VBC1Vi0ZbUeoOPPmhM8gsI4W9YFqTVB26QBpJIYvaTX/o5dvIrPz9e4EHy+IpXrXT7Hu6/i9p7MNGkNvzuuHE5Slia1Dr377GBUzWIuOZRKDTJnJXnblsZjRPGpl8Xa9RPOGDm3ZSUSkVhO0FNoP4EScbzXgwHJRvtzjxyt6Tq0toPl+q9mA+aRHx0HT7mGCR5/wZ1vPsH21inPTXQCt58NuW8SAR24kiq1A85yu+Qd/9ArLglnNg4IwaZMo71Rsb8iF5YstPYfwc3bqo/gDvIBqJOySXYeWfWx1KkuDecFCCl7AeRFZ5t5LKi0AsDsNUVOgB/CByq0s0lD9PYoGjEiwfJUYt0Vp0O2qCAOPq9ROIqSL0nIAlTsMw9LeCQhmn6uthJlbTE5pwXf7EuqSa+KW/l48sOqs5rkRGIRK7WYzF+t41bKHZb6Cu6ypxBFaSJp0AFpWCAs1RjtJJtzmzEQ4WY4CVh6+dyHYDKWTcIq70kdjBSkd2qZCFwy0pYe6YZ9Um96t8FnDoWqmZDYP8JpwqWVkcpW4EQAwuQz3EWDjdM1xlRj+OUyqdyJJBgrEZztMPeROxBY7+pz8RUt/NyhhwSfVKbNwJJQRYh8l4PnAlQPimh5OXYLrQY8uHPSypyAxvLrFNn1PuCR4IQ2xmMO/xym9JzZzBHtfxwd5qJpRwo0KZlQouDZaaouqMKi5fea0avCbDWnRtqOO9dXduzngESUH1TQVcwajYUYVFVxyYkL+MhD9Py2wQIQAAN9qqF7FPkenEeXX+gui2geNz2Ijot7puBDxRQk6s1F+bEzZkCuB7ecWT9gdZj1M8NU3lN0J7fHdb1l/t3Z48vQBzxZJ3uWrW2q32V/OoDcq1hynN6mKYGaerOvC5qURSUyTVRdVqrZtjf27Waz3+cLHfUQaoLpWKB/CfCU1Ia9Tf1z32OH5yRYtE5c2Yx3bd04mHTqj8WJBk3H+bI+/vHpB19sX0aZtetqieT1HyaQuYTMz+ZnDp//0dZQAdF7gr2+eAADUvYYrEsJgViPUIskBjg90oNgJbYQ26Mhe9KkHeZSwU0QEw1kZ22HqwuCupQPWOoF+1NFCy+6L+DvBGqxlM2xsilirJycREYMEp5POsKsTWavvCtIyHkP70YjpnE6GqjBouhIDH6Rm0PBhXZUWYHm90AbSXnNQcKMOeqiDAwpwmJn1L0csz/axXtx3RURAvBcw7DqkBwH4RMbplzZmkrnUJsBZ+FkoAIrY3+sqS7rOga/QyDg3MSDKpP2iXeK3M9IQrkOh0EddcN/L2CMaDq5gVm9rH/kKwtI7GFSgyEouzA/iu+ol2HKN/kP/xEDEDF9ANSJq88IjZuTwTMQC7QsQKVDPFayEpf8Lh7143iUSZqNhn4/Usw+QM1C4r8IDS5osv9NQLhH6RF7n5VpuvuZi/3yf/rscPdAik2AVTGjDez+ONdXjOI7jOI7jOI7f0fggM9Vx0JBKQ0gfWbyLuse87NEaisi/25xG6NAYhW7NUNYogOUYhdrtXkd1nsYKbtJnyEd5SI76YEVU4xnZfSWim9pDvqPr1xuJYpfqo2YOtM9YZMEJTC6TywYZLqcIMWR2qvfkkBG0YmvELKF97DF/uUHDrUOXlydY/vsy3s/2P6E0cDId0A4F/u9XnwAA9K+msJwZyImHnTnUl4ml2X4cQlOu84ay52qIaIAZFW4eWCxgSPUYAPAzB82ONuWDOPCYFQ6xYV84gf6Uvv7ZP3mFs2qP73bEKD6tGvz4hKDgb3enuH6Yx3LUr28u0GwolVdTE9uB+t8sUe2zWpr2mPyUJvJ81mA7VPj+W8rQRWWhGBZVLxsMT1Ss2erSRjOFoS9izW1cMUrAUb4AUDFTVwiP9qGGuuMM9BzRt3Q0Kmb1gykwDpoyVBDjNAgRSOVIRSkgIiY15XvF0Buvs+HUwZ2E1EzEOmxZcSsVt9QsFy0WzMq+vFti3JYQge2qXcz03F5H1x4I0PMLiEgyw4nrPKomeRHrufNFB61sZI96AD60oQ0CkuvUBbe8RJixEEDgEzS03qLzTeHj9+JFAei7grNdXksWUfEIIDWgAP/6wkdmsGoFhhNPXAwwi5wZv9WtBNsKY1gCw4UltjAAGAnRhXorZbSBAesKoHnJCMXUQrJhgW6J7TueBskjD8GMZ91QrTUI3ts6E7LQGZPWkyhJeLeEEwdZZfsHLbX/AMBDGUVG1Am9q5Ih/baoYnsQLEgbGezXOmTZaYaoC89wf0CWMqYxhIhIg+qpJh7uK+gP2+EQnn8fxwd5qIYhZTL2HZ3ETTNLJs5DgnzHTkOGftapgdkVkCxk72oHMU8QMMpEuhBtEtR2E4eOpcdkSxZWQcJQr1V80c3cQ/UJshsWqYWmfidRbjzGeSIMBMg3F80OUMpAnSdQPfdcArBPemjp8HDPYu3flfFFHxaAfkUX7KqK+uNCnebEwa0Su0o+FGhf0Od+8dkNrrjn0VzX9Fn4ULG9gg1KUp2KG6soHCAzJZ0+tTCMc7KOC248rgBZsIGk3fxH9Cb+dPEOj8ttPFQHp2Mv7Zu7FYQA3t5Q7VQXNqn9eEQzc+WA/vMuyaMZCbujORgGjf5uAslwqr7SsfY2zBxELxMhq1awPf2e3gvoEEvNqW3GtyGIiMYc0IUBRhlbE1Rp45rb7muM7C5jjMS4riJBy2nEA9FZCSE9XDBq36fWKoA22uCGIp90MQjwTkRHkODyE4If5wXePdDzHDclHeLhYOxlLNIWWR1ZjgTPhuck+xT8yT2RiAK87EsXbQQbWWE3TNMhKBDhTr2VMRA1Uw85ZvCgT4eIK7lGHBxtCh8PCjFkMoUTNlLnw6DappY0IQSE8ukgMkDJbUW24j5ShtiDbSB9L7X2SAOI5QDPwY/ayXiNYkdOL+HQ7h5lJMNtpg61cpCtTCpKa51cjBhWjQGLSIQmlxGkhCGbyWjG3ibDe185qHcVRDDsWJpopK60xbTuY6scBGIpq75NNnBeUV9q4Hp4mdVNLX1fpPgpDQ8Ik9YmybGmazqdHcDv8fgH8BGO4ziO4ziO4zj+fowPMlPV2kIWFmVlEgGpq7BfT6I+L6RHsWSyxnSMNmrjqGBMxmZQPkbdqCxlpkE8oHLRg9QaGYXUdSOIYh/ZiylqLe/IG7I/Da0yAvU3nBls/QGlPhcmD2LhAEE/rkRi66181NgU0mO3ryGv6TOM89R6U96LSKTozgmCiopNcxOzbnQKbmFw8ZRaPi5vV7BBs7UkH1R18/+z9ybNkiVZmtB3VPUONrzZZ48hszKrOqupZmh6gTQLdsCaPf+A/9R/ABFEYMEKBEGAlmpopIqki6rOrMjIjAgP9+dvsvkOOrA4R49ei6xlLjrSTTfxPOw9s2t3UNXznW8QE/o6FQcbOWYAcO2IWFmNUnNreySR6K+DOuSYEQgT6cAokPrDuECAwUJYsmOw+PU7JlpFqRY0pLz2BfLqLM5eM1HLXu6x3sz0GrbfO4QZ/zyccTUdl3Ltq+LdiiYgUdLMUNNNDAwMdLtKA/E5lFDoMI9KxPEj+/k6qRRcFbA7TBy9tAohrurHUmF5kTCNL0ZgJG0zJAcN105VQlcnmGd8Hxsb0c74XL29WGErsXvv7y9AlNT4BADGQ6aYM3xdPQghaZYKecWVyjFFyb/N7OJZhAnFpD6ZpA5fYUYayk0EwFOpTndGqxczkBK8kmECUr4fq41R6DPMI8iTGibEsYSBxzrBLwsDl0LxyE4mwZdMB1AqDOLZhwSSCxBqgvElBm1cFsZsogIh9y89sHf6XYyfGDcQV2KH5/K7zwOae6sv9jdyD6wNxhtf2L8OsAIhM2KTtPyjINA3ADsxZ4iNEJJkHupeRD12u7bMbJZ7upkPShYkSlitF9oase+bwlqvJ2YSnaAlEyMLdcuqwCiCVs04IklmFM/2XLWakMlm4LDzH3/y26e5qC6XHew8oXYBHz4wRtr8pkF66TF/yS7Xn189oZP+6m6o8SBwafWbFosdYf95tkOJhcHasTTACAu0brzmB6ZVrf294YUHPCmsCBSoafYxIdZFc2cPpd8AeTAz5ENlfoMJ/LADE6eZfFyTUPIUDKJNcF/w9/SjA34jcOdFUstCCiwTyTpTuq/0IQ2verx8scL77670uKoLfopiNAi9RRwmTiyZXZgt6QCM2xr1hwou52R7Umbq+GoomxvwZFrLBFSvgE4Wx7/89ku8vlyrBeBmOwOJ9MaQTOwCk46D0xNSX3eI0rfbC6vVyuYn3ttiFL5jmUgOHKf5kNdGpMQsSF1IqYQnROLFFIDqKqOTiXAWirH/wPIFDZ1HwEIWvTBx9PKDAx0sZh8zFFoWTn6PAluGFqArfo+m8agqr/fdxazDdctSqoOv8N3XHExg9gbxZkAle8Xtpi1wuE1AX1iyfq63EjM9J3088iVQJblU4MJG7r+8SagTjLCcQ2/hnpwuCEBZDPwi6bnheweYfyOL7x44vJA/iAzx5vuerQjzxiJqSordWIZE5QuEesKQtbxotgLt1puSgRtrZuDrAjObOKBRgbxBCdQVXXDKxw1uyewvgf6N3Fyj0e9mPGk4epjzommlx2o7FDxR3Zkym7mcs2ozla4BcZkwStg7DaQa4dAkwCVmbgOYtwO2ormftSyTW/wvC/ldwrjI3zmxUxvkmk8coQxKsARw3IrKqTh6XbK1ZAWMS0Kc6FKjIwQ3xYt/nOOTXFR3+wYGLcbOaZ9n9h/f49Wsw5NM0E/dTO3vNqsZSPxTmQKeVI5Ae6cTWvvsgDdXK9W6flidlQrOpglRgyuzKRGn5qIPfkHobhKaR5mUD9DJKLSllwrwjawem9XxjRzapBOL6QyqB7lZH2qMnw8Y8044lXDt9s6AslzlOe/+c6SbGYDDa6Hstx7dUOHs2U7PaV6kgACaDXDX/Nq8GXD7wOVAuGsA0R3anQQz54nClIc07S3cxpYecRsRZaLqrwGXq6YLg6++fqGpMrAJyL0iw6Sy3DdsFgPsrOg+R/E4ToFAhitDABg/63VBdysHGo1etxTKQpc6y16353kmI/VyNnurfq167a5EbuNi2RTVEa7yKmlwLsDJcTQmYnMoYeOhiji8KKQ0HU0AInEyCABUUd/Djxbj4HB9yer7ygZsRn7Pr7+/QS3i/fE6op6NugGM3hSDhL1DtSX0z/Puh6PPAF6QcrVoegIsVNqSJtfTt7yojkIwM1eDVuEgJtu4fL6oyFtyj52/M2H2nTmSamhcmk3w5wlJPHir2Ygo19fcFUvKOI/ce9wVBAQa1m3QPAL2ID3zM4KXRTVRlnUV4k9BdFLZdHUW7cfi9zv1Eo6VmE/I/WgOpvRKPeBlfxrrBLeyJUS8n1RvBEQqi06sSsTduEgYRVKT5p43+hm9qACMufcNhGXA62di5NLXWhTs9g3iQ4PuuhxX7s+bkeDlWvt5gu2gi2wyBUEwA0CpFAUmoWymJvIaSB/2hz3Xaej6j3WceqqncRqncRqncRp/oPFJVqovLzdwiwHLuleHnBftBn91/xlW2WIwkRplNw/myFbQDtCdWf8i4Cc/Z5u8RTXAUMIv/55TSNCZYuNWJUSpasglxLcdzHcMu57/PdRhafsFMPtAaO8LnKeJEpZp/1nwHTKsBt71ZVOIcZkQzoP2+2a3pd/kW4BSrXBNfNFrUs/hTSxZrQTAJBy+EJOLxYhKsFoiYL2elRzJuVeJDLmIm6stXi25Z+ko4N27a37PumxD/fMRsQ56DmxXHKDswaDaFvec5uaA2Ru+AE/3S8x/xaiB/9UC9NkAkqBpsgm1MFj/5Nk95m7AY8/X8/36TAPpiRKGzOT8UPHnvGFYdHlxwOZwJudb0IUMEXZWK+H6Shy2NoKZjqUvSKFApDb3BbMMZbAgqSQXZwc4E7Fa8TFuDpWGu89mpddlTARtLObiujPOgfE5n49qNmIxG9RtaVZ5vLuVwAGb8OpmhcYVVu939+Ik9aEpMpye2NwhF/lPlfbB3dZyOHjui/dGM3yTSxM3IYYf8zV0u6lVJSHNS8h36KxWwoiMVmgA9gSeBaAGD/WdPUpiGc6SngMQQDaimWTR5iqcoVv5rCbArCplu9u+PBfJ8rPh2wmcnfuQvbRXFuXAsqwrVaXX7bYEv0x6fnDmQY88h4xXgeHhPhtzlPu7XxSTCJiEAKD+vsC/yvC1ANIE9qWSp5wsSiti40CRjgzs82v+PACe8O63LBMze1s+u+JQ9T6nLTWx3BN96XXXTwZ+UZJ1kpEAeTmO5CbnzpT8V6Aga9mNTFGbyH355H/8peonuah+efaAelnjw+EMC8eQ4Gqc4W670CSKcV+jkpvGdYCY9uDw1oMW7BsMAHNA3WAA4Je/+gztt6L7HIvvZWipuNK87REPDu0Tv//hBdC9FNehdxazu6g9uHFB2itNlid6tT6rk04etif0N/L/hViSCRlhQlpCYqJUXsTdb5sCwZB4DYMflP3rpHd99W/nCkMNFxEmErJrG6UKJJOpPwu4T0v8/PKO/50Mrp8ztr3dtxhFvmE+NKjX9ZGl2yh2fdXagoKkuAC4mXO8GQDQwWqKRvOLFS7qEQ8riWOjhM+vWGO6rHo89TPs1ac2YTnjD7v77kI1sf4sYvF2o8krm65Bcy3JRecWJhCikJjIG0Duj4vlAR9vzxW/am4O3J8GEB/q0n+WjU/e4KR5UOu3rqvgXIR1mUTGi2n+LrnfG29buBHYvxKYMUETZtpnI9p6hBcZxKZrcHHBG4SLWYd5NSgh6W6zUA0o3nQY1jLbRZaXRdEkmlg2jf7S87F3xW853zs00oRMxotEvufqDS/+ABOOUrKaNERVkdS4tYVfBiQ5X25nShugAYxApLYDQ8PqkS1xZGBrz7j06EXSZPZG+/NhEVXeZR4rVFvStkt/HZFb99WaUG9IFxHy7EcLAHZI2L0yyi+Y+mojQSFYJuIk3kAAoFUF+5qvRVsFHLblWQvhGNadbk4W39gj0qFfll4xh37LZasn0Y/E54D/yMA9FqtTPytuSzCA3VolZMU6YRT3qXRgnbFan24nrliXUdOVhsuIalUKjcNnodwT0n/PNpTJFgiZ+RVyGD2h3kCj3oxE4f0xpNWc4N/TOI3TOI3TOI0/0PgkK9UuVgihwtv5CquRd7d/e/cC3aFWAoW9r8qO+T9Z4eUZl5zORPxk+YDvD0y+8dGgtbzT++u/+wKzbyr13p1WiEehxAcH8oTDW9nResJM4J7ZHYvcw4TgkKvYZBNoHvQY629rJTSN5yU/NcwSxpcDxlzNfHDo8y67TnB7Uir+FFJr7gHXCRlhRkguwQp81d4VsXf2ElbGZpp8z71BOjT4l+PPAADziwPm4jTUNCPSV1xVuomAHmASSGYJhybBv0xIsvM+DJUSidIsYPaWK99nyx2++XiF+CjhBmcj1kOhRN7t5/rz8+UODzv+N80CghDUaCD4v77Et88Y8v3izz7gn7zgTNmKIv7nv/kF6o8isXnp8ezZRo/JPFUwL3mrfXW2x7bjz95uKt1xuz1xlSCMXAKQBDZv2xGvzzaaIzt4h1Eg6u67JZrHwthMDv8giWP7NMcwd/j85ql87y2/X+cdYiIMUsUakzCfc7Veu4BVNrEPBmlVT0wSojLY066S0jhfKEyIOaagHIkh/Jw6EppCOKqfCN3rWBjoQ5EfuCqpgwAAIABJREFUhTaCIpWKCxOpxkTiYgJXXOqhDKgkLVUJ1FtlXFMgJYYBpaKtdoRxMZGXBXYoA4DmMaF7VqQyzQNpBbp9yyhQzDByhPor0yRlKNnETOAsqXnW42KZgzgsaMLStn2R24TLEWYjkq5bA9sdO1Ll5zpXyCpjskBsM5ZdSFAAw8IHy//25wVntRuL2Qej5ES/SBrAQAQ9HwCjA7lNlKhU/0hSPcs/3XrSHksFUgeAw8uoCAW8gVvnUHg+jxluN5bnvT8GotInuaj+ZHaPZl6hjw730nOL0SBsHcyeL3o496h+wg/EF1ePWFY8GW3HBn/z+FKZwW/O1/j1PUsT6lsHCsDus8wSBKpsoN0C4VoYoHVgiU2OY3swmN0KzLpP8C1hOOfX9q+LnhURwKrC2e+KYfr+tbAVr0Lpe40GGIyyHPtnBZ4xB8Nm3gK7hrZALqEFooQZb37CLNUMB8eqmF+TF9Pv3COlAknZnJBzzzhd98xhn6O8Elsc5mEGUgnP8HrEtUTQvT1f4+EwVyZ2U43Yd/nDE5Ziodd5h3Fdo7mTReN7iw/y3vvzA5wNKoF593ihLFuaOGmlJqH/kw5/+vaWP8t5/POLvwcAfD9ewjYBwxv+3XoxaH9y31dY/HSFSlJxNocWg/SVzcRJa7hga8AcKj4ODr/4khft86rD369ulFE8q0fc30tqjxfWLPj6RFcYxM15DxIYPe4qmGWClYu46lvtfbfOo7EeVnD6s6aHkQPrg8O+ln7f7eLIkB5zaGA5bGJJTF4BbAKpM1JS2JYOwvyc2PBlN6Qw595r/t0sq+LvVlyN+H+U+6x+MkV7mheSyWFmdjCNvADqInXudTNY7ajc67PE65++V1IL0MMbhubrd+JMtYYulrFJR/1jty4a2ehSsWE0OGKLV7XH/YNcT8sbEyf6bdOX58fdV0epVcmWRdX4CUGemIGrm4kmqg47M335oHjDkN/fbidM45FweBVVGWB3Rl3ZSJQA+Vy7rpy75oGQZC7rrzhFykxkY25XzoGfl2Qdet5rVGW8b3SDUG0A45O2uZIjBAOECbv5xzo+yUX117vnqFDj+9051iKi361akDdK0Lh8tcE/ffktAKCxHv/X7ecAgLuvrpEWAbNzXnC/urtB907EXFcR4xX0KajWpshfllH96dLOwu2Mahgvf13E2ZvPDQ4vEkL2aPWkuzvbce8n/+7qzwKS7AJNFRGlMrVbg+SAkJM6qEyK0SZgbxGyxmxiB+jnpESC+skcWaINF0lF7sYT28KVBLOSPtFzXyqTJOonA5/TVd70quWsHxzbzuVWXRPw9pwXVR8NNl2j9nnjaNVr+frZBpUsEvuxAnVFftBfF2OFzXomExn/+2zRwUs/cdjUZQOSAHQWX/2ffH3DLGH5z3gm+S9vfon/+i/+Ev/66QsAwO1uiWcznj0qG/Bxs8STWBrGbfH7pToh5p7qIqKajVqt/xc/+3/w04b9if/19ifY9TW2K948mNsGZPNkDVR5x09AOAtY3nAZ6EzU/i3NPC4WB6ykn+iDxbLh4x+ChY+1puxUJmjyzXrfov+GJ3wb2UBBSW/LURd6GNZe6qQfScWoZii9+WSZkDJK/288T0f9s6kpCg0GLic50URzC16k1NAklt6c8WLJKa/588LCSSmBRquSpvrWYfFOX0R/XbgF/iIU6dPBKmJSrwl+bvn5AC8MuX8bGj7QvMAkd1xJTour1ESturunVglZ4WDRfl+hltAnM3KyFJCfAdL/H39g85efyZSYs5ARHXMwwD6bfiSOmgRvtvPiVd6fhx1YD5qPMcxiuRaGN+dW9fOlClfjCci9YsGRe5CNuUiYKErP/UYMR0yEv+d7s7m3aO/kOLqE4YLUx7i9l0r1j8D799RTPY3TOI3TOI3T+AONT7JS/ZvvX8HMeYuUw53Pr/bYVi1e3nC19HqxRi9K5cdhhtWWq4nqxQEXy4PCOumxRsqUdBdRva+LXVgFjC/EiL8jFc23D2xOXQlk4htCJ/Zl2380AJHUsD9djkg5H7Mz6G8Sxpdc9dgmKCMv7p0yTEEAbvri/hMILpulRwMsPIaGt9qz7xxsLNVAHuqKkivJsbAVk5N+iobFJq3OY5UAM7FnM4lN5cGwq5Wqu96whKF7XQz1/+bb13x+ZgPG0WIh/T+ihH1XePlrgYLXmzlsTzh8zufj8tVGoc/NvkV/qEBy/Ie/vtJ+WfXigFpYvMZE7DYtKDeWLwd8tWK5wb/Y/6f4r978Ff6bt/8TAOBjOMdverZB/O9+++9jd7tQFCG1UXM7aSSF+t+8fcBVe8DrGd9X33WX+N9uud/8YXWG7n6m7xGrIq2wO6NShNBG2LNRJUG9rxCEPWvnHu+/v8K5mG28Pl9jLoz2mAycCfhuyzKaj5srHHaCrT5VxVJwHoAqwYmcx1WBw84hLOMJIkeB0N7m452wOUeBDuU0Gl9kHMkJfJotBldGf2YGa2G/EqD3kh1Ima8ZksyvUZgw2DcW1Yq0gmseCyTbXRdJhx0Ac28VcaEALTO764TwagBW4rDVF5aw8QAmlR9MQWam+Z/JJr5uGZY+GGVNU+QAjeahtFNUThImuaI4ApbgDiUYo78OMF2Bct2hyNAQJ3KV/Jzm4w/lOsVKTrJ8du5Z8x+ydCZfbz9LSALCUSpGH7BAolSMViLpc4wklWq2bt1bzG6Nfhc1Q6k5OUe7CjmDdeKw9GMdn+Si6kcLM1q0s0HDnVMivHi2xss5E5L2vlYC0tyN+CdvGE/6zdM17n5zrX288SwiivTD3FUiV+HPiVVS+zFEqESHAuCGQvxZ/YVXSIq2DqmJaD7j4+gPFWbf83tsf+aZQCLSh9CZksYBKLEHbYA1kSOqALhZUMejNBL3FgWW6l4H7b1Wm5KcA7DWME9mU01ZSgSKqTjVTOZdCkCw6egc5JFWtT5E+1dMGFEv3d4gZPjuzOOLq0cY+cDb3VL1p+vNXIOryUWkK49qmRcR0r85m3fotjXMLV/f4TJi+QUvbI0LWGXYNtRIj3WxduwsPt4zaen5zQZdcvhqYD+8//7Df4C7/UKPwyxHhOzzPGMNJMAbCSsEkk3XYAwWtzvehFU2FHWTfPUcg2YXo/YrY1+rNMOcjahqj8slw7/3q4Va79FDCzwf8eqMCVSt9Qr3rvoGv/7+jeqJMRaCUGqjEo5ACWQTjJDB+kNVAssnvT4AcCujtnNJdJMAw73jtVdSFwVS6VM882x12BV/YvX0lV5lvg/c1qhN5LgsjkRGbLVzP9FNFoNqzeQYjQ9rCX12KKqS9nabR0K1TRhE5+1nxU2MEuAmG+Isn+EXJ98XvLikrOM9H0p/3huE0cCs8s6iHC8uBsSHWrWuYZYUXuYFpWxs48RTubtJReeZ3bzkq4+LVPTQXZEDsdNV+Q6Jihev2xGqLRWdenV8fesnM7GGnOhNe0KaEKZinRAuhGjZBm3PkBQD2Su6/UiY3UlLYFG0vsnKPLgvnx0dEE+SmtM4jdM4jdM4jdPI45OsVONogcFiv1modGA27/GwWmC95wrm86snNFKpholBZVt5pIXHeMgVKMG+F7MH8O4xpzlMcxen6QvJMNyz/1lOk4ay99zLPZbzHo93vJ1e/F2N7rkQK85GxG2F6rEYPuQd/3hTUmTC3iHW5QODs0qcwYqN8aucU7mIaghOvuSDIuHICQVUIDiKAKiwCylNZBDCOszHRaFUZKYnrcjDLMFe90osqhuPIIzTlAjrvsWrBVeWhhI6SW/x6xpmwR/WzgeYs/Labt9gVvNr82rE7KzHIEzMpvYYhdzTHWqMGd7cW4azBA6285Jmcz3bY+Xn+MazI9R/9uxXuBMnjt2LBs/rDf7NmiHrf/P+NQZhYi4WvR7Tdj1DOutwMRNpRbDaOshJOk6+T4xUiErnIypxCJq1I5wNeNoytBHezeH6cv2un6+V4TtECz9hpjy/3mgwfH1RAh78wU0YNgTjAgeVg9sFWZ5BAyFVSZ22zFiyOSmWcPdUJZit1ese5hGUTRDWDvVTgS2751GdkhIloIFWgxQJNntiJyj7d7hgtmmGK5NNii6MF8weHS6lCqKJf/DEdSw6Qn9JWp3ajlCvpfpyUrVNJEL6LEQgzhLCWfE/zhV8GKz+YuotaDBFepOK9Cb6GjQShqvC6M7zQ7WBljeh4SpO0QBDmrJDiSHxPJeQL6TF8SyCZhMkbE+qPEi2VOTjeVBiUj6OLMvLfxtr/ve4TEeEpzxY3kUMbwOMemWf4SbCbK3+nRkLgkAR6C8nrSZTXjOeK9nwR1DmfZKLavO7Grat0b0dkSTNfj+3uH65xrU467ycr7GbRLwN0iy6ag94F6/1pq+2E5ir4pvSaHIJdOKasgWz4X39XiQNywTznN/w1dUG33z9DPOvBeJtJykYqxrVysDts0ynTCzUBjjpD+ekmNx/CgPx4gEx6I7AeJEpnEndhWxfkj5sRwClI7el0iiVhfUfgmpI4Ks82brC4EQsvcbl9V4XUYCDuHuB4nuqES8JCawFHoPR7za6yKxesK7OmaivGRNx0fB5vNvPUbmgkX2WkgYdHHxdJs+LEWlbEmxSJIXNv/p4g8duhquW74lv95dYDYybXdQHzMyAvdwjdTVZsP7qEuGnjLe5dkSMhJWwzAFo1FuwCcZEhLyYTRZV25TEmqbyOGt6PAosjalLTZXw8PEcq40suL1FJb3RsXMwLmJxxuckBKPrqJt51MJIHkeLGI0GmNPeFVavS7D7YlHn52XhSbbITMzIG7y8oNidAUmvuHnie27/Ruzvzr0yga0EJ2RWeGiLZVByZSGj0YB6OoIm83NgBu5/Zg3oNNSeF29+v93nvJhnGDrWKDGKwJHb0jToPTRAWMYiJZoVBrFrvLZWwsjno3os7lMaYD7iiO2OOEmTsmWhHC4Skk0wE/a4URkKS4yUldwmBIFxzUB6PmzPn6N6VlM2J/bBHlkwVusSv5aIF9QMDYe62CDaQ9Ht2gP3Q/2cv8xYW5ilbAx7C7cnzN+X7z3OSx9Z4yjPWS8fmsIyj44Qw+8v4j+28UewLziN0ziN0ziN0/h3Y3ySlWqYJaQ2gQaD9g2zJhftgD+9+qiQ78IOaERs+bvtFX73wMwHP1pUH0vZeXgVj+AYty2QjDsc5wxmWDgZgA7FUWnxZqNRZN/9fy+x+L4IsruXXrc+bm1/Lwoqm+EbSvBiPqBZpJmIEqE762QBf15KzOZ9pWSB7kVU9iZFJpNkCKxalYBuiryjVxZl/MGO3xOMVD3DTVA9m1mOWMz5F723IEoK/xIRTI4bC4TNupy4fnDoH+WEGGiIMlHCohnwYsmkrvXQqItSW3mMJqk5Q0yEjVSLfrCa55mCAepYvHmHWqu0YV3j/bbB5rLVz8vw8n6s8HcfX+BGyENnbY/dr5llO55FNAv+nnXlMatHhWdrG9ThqBM4OrO0B2/RZU2vtxpYPgaLD+8vQZqxWZivBgZpMAhSfVVrCz/n962ed6gnlZS1UY33w2hx2DLltGo9wrYq7HE3xT4F5s+ZuKa8FB205ElNhF16fTHUDu62PCfdTUKUaDb0hjWW+T1sYQYXD1yBkAVWNFKlasTYiOLCJJXXEeVmUvDkKph6A7sxStqxByoM6CpxFSgsXG9Lu8NfBM5NXuZAVVI0oKqCmop0j+JKlYlE51GrO/I0lX0i1gVaDW1SuDr74+aKvHoyaD+W7xwK4AGKUGJVRqD0/wcA8prbAbHKnwVEU5QHoaWcc4BQc6WsGcpxopd15f1Cxb8T8/lwCemJ79X5dxbzD0mdkobzci38HHqRKOWggkJ2nLpF/ZjHJ7mo+mcjzMyiXg4YZBJbzno8DTOc1wyV/Wz+EbdCE/y4WyBkg/G9gyMAMl+4HSlUg8Q3q/Y5FgX2ncKluceRJ8bDry80n3E+CCsxM4onTinap5T1xi8jqC19nszYpJZjUpJkuVKiYgu3iEDk9BGAjzdPJOGsBHL7ywhsKzXsZ3H5D3ovk5+z2UMUxuB4+Q+wIwNhv5kkKxP3RQFesLJxPQLh1U/ulcm7vl2ChG1p6oAosDHVHo3z6nK0BOFAEloQDSob1PnKR6NmEraKJcO08aBZwlhJmPxo1I2nXhPGM4O9QOfmakAvJvd/9voWX54/4kXDrNv/4V/9R6gF5h5uAs5k8X19vsbb+QofOr6XDr7CtudzsNrMEQergdGhc7ACxcc2ga549t9vHex9pZuyKEkg+bogcqA8IE425/zZF2cHzKoR2760Meo6Z44OaOUYH9YLlkJI/5lGQrUWw/stp51kOZJvE5A3U2nqamQQdg6U38MWJ7D95xGpCaCcarSZbNBe9giDAYlVnhlJjR0oUjEiiAz3K1uVCE7CKkzgBUHv43ZqNK9fHWkeWMohm88wT+xEBk5GChvS7xnq0sIwe4NUEVKWubURg3yXuBzhhY1P4M1l7l+mKmnYOwnlQRe+QPDye7Eu0LLbEexAnKMM7q1mqUxoSkIVADFdKV9QP9cQL5r5mWw41QcQWJugWbHToHkz8M/jjXxgb2DFctFNoHETeVNHwiFIB4f2vUjl1rk/zb87NY1JBtoftnuS+6fMnaHVIKMf9fgkF9XqYwXTVoj3FegzvnutiRij1SriblzCpxIVFuX/057DxVP20vSku0V3kAovh3/UKDuzH/QfKbLzEf8j6Q40tMd9yOTYRSWPgEmKRxXLnJEAK5VADIR0sOqwQoHgLyUhpGeNYG6PRjtJsKii9vTS1qG9nQSphyIVCrOklHiAHzi1p2t5Ys+B0b5zWmGlOukmII0G1Bv0Dzl2BLAveBGZzXvcrxYYD5UeV660ozfaO0uJ8HG7wKPhXca+q9Uj+PnZjh2X5Do5U9Jg/MShCRa8SGf/00hKIImOe3xZ2pIScCOB38/bLf7Z+df4b7/7pwB486MyjLpUhDER/urjW+xl82ZtxOY9z4xmOQImIUhajJl46dbP9+jv+HvZAy9Cw1UWQJPaaZqON3V5QWle7Usf2URsugadbCzaesSZEKbW+xZrQQOS6KJrIQHRiGMNZVsmQwqlIkIEXF+cdEJDiLIo2VlAUHtKAnVW7/dUJcTng16XMLrCE5iVPi35QoYzg0hGpkSizBloREYylW9p77X8P2oCUBGGV4VIpBF0lxFdKOhIbKLqXrP+ODXlOTTyc/AGVklczEmYRqmpTlW0ohqsPlk87MGoF7ft6Uhm4ufQqDcz8DnPFoCJiiTFz0uP2XYQwSv0s7VHmwBQmizu5Tm2A29QMsfCeKjrG6iQEYcLlhSZR75vZx+M+p0nB4wNYfeZ3P/zCJM3RhOOiRllkVaiFc9F8Y+gUj31VE/jNE7jNE7jNP5A45OsVP0swcwS0sLj+QU3Fwwl9N7BEu++fvn4RrM4x2CZTQugvWcf1Ew7n+6Ys1RGK1VXoI9p8sRwkeCv/dEON4/QciJJ8VSdQFlBGHhmsiMXyDSZVBxOxJIlCHOPBkJ1n/FC/oxMo08OKm9AIGWANh+ZAaoVbYWyBUvgHWauFGYJyZT+UKwKRMjK83Luk/Tt3NrCdoVt7L7cohGDh66vQAQ8e87Q6nrXYpCeKvUG7tVer1lMhIPAm03t8Wy508+qbSjm6YD62Y424rCXfuWmBkIJ1DZjEc37M+6lzS9K7MbbJW/JL6oD/vfHnyuy8fzP7rCoufr67e01erl3vrq9wbhuVIKxuDrg2WecKLPravRdpdIEMxDCa67W+/uZ+hMH4CiFhDxNElqAeDbg8pK/t49Gv+dhqLBse4V5UyKsdlyNDX2lwfLurkL9RFqJTJnq0QH9i4AkFV310ZW0JTsxACGR1UgF51wAibmJX9cwfTnmcOm1UBs3NczOYryUcqmKqMR0nkI5lugSOxn1+XkqPtVIfM1yVeXPopozzK4O6rK127UIO4fZc75/fvHiA17N+B77H//ffw8OxTXI9AZRkBNXB/jeKephq6gsVfOhUWYtRUK4mHR2e6PHm79LSZhJR7KTPEKbMJ4V9nL7wapZff77DNfWj6V/myYuTxTkuZ5Uw8r+7QkAFe5E4Lxo/fwa2oo6GlT6ucklzH/nSoXbFfnduACG66jVujkYDTNnFIh/z3iuSjX5yosLV//7H/1jG5/kovryTz/CLRoQgHORYOzHGmM0uN2yps9HgyZb3j3O0f5WnJcM36xKToplYQBxryA/7GZLSopg/Rf/HFu269MepSm6uuSEWJGhpkBI2T0nHTfyzcEiSg/LLLxKQcLBHUG5ZlcpYWI8j7yI52c/lgWcqgj7gbEmnjALESI7vfBrx+dzGj484bcAANzZoP1LrOoCBSWGrjLUur1s4bP+tB2xbHsNfzcmAQLdfvmzD+oYtB8rDN5hJb3SPjklmgG8GcqL3sNqoZKaKSmHbGKoOuTeMRBEHrC82RdjebBk51f3bFP4m6drfHnxiD+7ZHP8kAhfrTityLmgOtWwczBzj/MzntV8NHh4ZFemFAxSpAJ7LzoMIstpzg/YiusT98iTwrpVFZSAFaJBTKQQ78XioLKiPjisDi12EvDuXEC3zzchaUQZJZmEp0wf+drjRUKiIrtCLDBrbGKBGCPBXXd6AxiT0OfQ845TXRSKHQ1MlnF1xH0+jZOzSnozPQHybEVZJNyh9AJV+tGKCb+8vb3u8fbZk5wDi/e/ZdvJ6skiPBtV+tRaj//1G7aMpG1ZJACA3h7Qyiav7yokTyA5Bb5zaL7h81jtyuI1LmUTnb0xJ9aBoQXiLCq3wR5IoeBhOWFZJYLtSB3b3MTKLxGf3hxxF9rJYnkoz17+fb8s3ycvuKFhboadLPbjopxHfi5lsfdFApQD2IESFJL39sNVISpSEp6JFCH1itA8Jj3eXHC4fZJ+rrzHubQwjthmP85xgn9P4zRO4zRO4zT+QOOTrFQtJc34m46n3UxNyxezHk9rxibqd3UhDyQAZiKQbtKREb0dWdgMMKkjV6rGJ91CNndsWp0rP0Sglh34cBUZ6stM3r7kMyaXAGNgBP6JtuzwUyCOH5PPRYQSlcxYYK3YRqSaMAqU6C9KVZx27gjupQnZYQpdITKhQSU146TSJoEn5VebdsRepBvUkzJY3Y7PTd7h3vwrh/6Cj3/75wN2pgUJexnnI37+U05K/cnyAV9v2eHoMHClmklks7bTaDNnIseq7bjaC4MtlVikEpm3dqC100reXxYvYYANE7qVHP/B6jb0F//4G1zVB3y14SrorO61gg7ewlX8/vX1CKKksXMhFMNy146YtaOeq76vlNk8ELCUgGtnA2ZV8fT10Sgzel7xsea838d+jndrNs3Yrmcgk1DliqsvuG4KpMxyElMCRV+IDeYB8SWeyFxiDY1HpEQw4hsdluwvneVO3bbRv0uGodvMULYHOiLuBZG3AVydTtshSpSL7CY0lZ6ELJWJhLgIOHvO2OIXl08Y5Z74uFlg/pzx08O8QdOOmInpxS9vX2P3nss04xlxSgsJnljVCGJukBEf5CDxjyVuMFoU0/+Kv2uGO6MtUXipijCdQb3KUH/5O3+W4FZSmQrSlecGP588WwL9aoiBR6ns6Jjk42flNZoQvKJLSjQEgCoUxjNFriyn0pz8fE5hZ4pinCHnoHlKWnH214REhb1MAeivMkqWJteTPYE1GnP2+2TOH+v4JBfVs6ZH1STc7pZ4lCDsrBPNfZOnpwXcdzyZ2gOhvxHoY0Wo1wVioQkHvNoyiy0/BLPbAgX3l8c3KzMWpUfTlbDuZMATTH5wOoOUKekNQ3FHzZLMUOwtIIsvthZub4oEJ1KB1DpmD6tjTARIJgtcjAjZKPxjxRCRvGWsf8gYnEy0kzxMGJ78cs9ptyqLI4cg85uMn41IO4fqqfSV82vVfETwBiTMSdxEDdf+ZneJVccL5c1ij8NY4UHOweZpjq2wf6vGY+xdYTN3Vlmly8/W2OfFtjWggXhzAQAuYZRFdHQ1Q5WyOalXBv0zXjTu9gs8HObaR/1HZx+wm/Pfdd7hURyOgIS2HlHnxWaoFOI1JsHZgK1YYw7bWnuSdeV14dxsZ9hO+ughGE1XymznzDbu+4rhcgCu9qjroOHpcbAaFG62rpjrVwAmSSaH54m1mQDn+W6sQot+lgq+Fbg/CgBm5hF2DnHMTObJ4qjQpvxnokfMbYb8LJA3GBdlEdEep+cFIf0DM1ZsIi5fbLBo+FqMweLNgnvf/+H1t/hmzxrzva+xGRs8iV55+/6suCvNmGGueuWmBFLEzjJ7We6DME8Ys450bXQBjDVzELSf2NGkn2hhJkqB0ELvx/Z9+VLJMgx/FFYhv9fcS98xP5MWulgmmvz/iXPRD0fzJIb6co776yJ9au4sKED7nn5R4Nppyys2vKF3ssi2jwnjrDDmY12u23gVFK6uNsUNDsRzYihqLz7mk6Tmxzneb85gQ4PBO3SiO6xrj5QIvcg4jE2IX/J2q5r36NSMgI1Ksy6z2nHgLsAJDMNlRP1UhO3DRFemPdUqSapEIQtp0/9A8JMepQlQ0k8MhDiP/+DEAk9KfEpWJqC8oKBU1iDp2+YFmKAkGmMTkjwodgBMXxZfFoKXPkyok24EgGJ76DZijygTaNV6BOmHhm5y4AcHezBlMUulZ9PWHr5vYT9jeODNzQozx9viD/slWtGl/mT5gOt6h39JPwUA3EbSSs/aCGo9+lte3ExPaqG3eZwDObklAkik1b95skrwwmzk3vRWrAhXgJ/x+3/8cIGXr570uH759AbbgRfVzaFBK9XQs+UOjiJ2stg8fTgDSS+z+myDMVhN4Fm+6LT/uVnNeKMERitgoItgbKMm+sAlVLNRq1Hngto/jrsa46Mpi9osFCFgKsklzQNLonafy3VaRp3InUyEeRH0i6jHTwlaSpr7FjaVyhKY3NMdWxhq8sq1101ZXjxyUhLfnxlVmdzDUuH/hu7fAAAgAElEQVQXGZcpJLdXWwzewlm+v2bViEPgm/O7w6VaSa6GFt98vFIJU/VgVSqUU5twwdeNIqk3s90atmFUP+GocX1Tyz8aCSZCk6NsVxZRv0wYJ6hWtSmyNkzkQBpAnglZIzQc3fYJoJL0YsLEdIXKuZ9W9/kY8++FBkee3mYkuFW5aFNyGFB+9vOkc4EZuFjIc1t0Rn+P3z9hFP0vzTzsmu/pdlJkxFqq03w9O9G4T/raP9Zx6qmexmmcxmmcxmn8gcYnWalufn0J07ZHUEPaCFz6lreF7WyAl77j+naJ5nveilU73vVlsf24BNZ/UTjxdLCaSUqhsBUx2cVnA3D9bFOqtCjMPN3l92WHHBaBt0ETVxZl7gZClN9zGxaTZ1g22km/wgv7+JDTPoB0KWbYO4f6TliZh2xSDv098pleeSyxSQTtiYUFGz80Sz6P1kaMArGbrT3aiZqRCnztklZih02Lej7ASYXbeYe/W3GmKVHCf/7TvwUAPKu2eBznyuqNkdDOiqlAN1Rq0pHmSaFm8lb7WQBXGGq9VyeFw6yLCJsKzQN/t/4q6Tklk/Dh/SVuLfcvUzDqjJQmNnaPZgZrElab7HyeYK6LixQBamH48PFc0z6mLMhUpSP7PvLEeahg432gtC9isAirwrqlVOD5FIoEw+1I2efJAv0N59vmY8zXPVeDxTmpOIgxZFuuX5hUOLlHnYefJ8RsBhFLz50Gc9xfncXJPRFh5vI3CxwlNI3nEe7NXr6zwdg7tYy8aXdana77Fh833Dfd381Bo9GUlmRxxEimuS8OSOuqPLvCzs89xfk7V445FYOEUIl5hUrsEsZLOX91+kEYRoFIyRdzhvzvzD3ggI78AhDqye+FH7Bz88/iapb7mlwJy48BoB567etNYr4HAIoJ/bnBcCGnZNID9WdBK/BoE8y2VKcxoNyv5jiMIPVWX2PoOnMv2JR/KieaGlH8mMcnuahOF4v8AHSvR4bYMk28d2havuJ9FdG/4p/Do+OYouxqsvQFSnRMjQ/z339YpgunIe5xKrSySBpUjVQWKH4NRUdqEk94E4BBXV+qpHIVM4C/S26L3YTirdrT8WbCJiVgTOO5ct9kKp9R2DnxwxkmAebZQYkaDuGet6W/paQrXzYB/D/K+Z/2kVw7IniLxYwX5sNQac/wz198gBdGxm/2z/B/f/gMT/dLPa7xSWQoc49nNxtc/QlPtB/uL9Qf10xgubwxydcMCZq4EQ4OZm9LCtHkXNBThWozIdRUQPaujFXCcFVg6G5fq54YTVASU174Dx+l/+oSaMavpZ0r1ykCqQ1HiUFmksxT1V4XVWMDkCfy0SCOBiRe0PWH6kgKpnDss4SwiDpp4vCDCEBXrj2Nk7g3A9VVJAcm+WRYNzkg60bniV2JJn1hEuJWatjCMD9D5EvaiieDKCu121qQTfDSd8f5iPMFs2M2+wZfvnjAVSMh7t0C3z5cAgDGwanGHDYBE4/qMC+SIHM28oKafbNNKucjAPXeoH6Sy+F4gwUwsUjJiARUd+X58rNCVHJ7AxNKLzM6aAsJwARa5Tkjw7V2KOcsOpbhqB41TmRuE/jXhOPFis+r/J4Bqj10oYsO8NIPHc94I73/E/5wOwuI4poEAySUjRX7icuxN9MFPcFtCNjmyYHbCwAv/DoPBf5uWd+e/98fA1npBP+exmmcxmmcxmn8gcYnWam613uYeWSXnrx7DgS4iItL3u2GRNjccgVk9gW2ZBF7IY3QqlKJAdvKlF3hFOZNLh2JuP2sNP6naRx2ZxhCzeYPiQq0Eol321li06RCPPFGq2ImIiTNQoRJSC7bH1mYbkKSiKTwlZnAoBgBkwiRys5aafm1ZD5m4kwFNQRHZ5DmAYPnW+uwq5X9l6oCoSFmD+HM+wcuv+RSYF6PeNjOsRAnoMp2yoSNifD1jiU1//b7F/D3bYHOIgqTcTZie2hwWHHl6u4qNE+lCp8yqIeLVGRKbTqSJsUmwoylRM2/V68IsZ5AXalUDWYg4EEC0TdLdfcBACJgEHLcsK/ZgziHIowG9dclhkQJOxWAaPW7xSohSWBBVXuczzuViD3tZ9jl6kL8lTMDFam0LZJJCssoWzSj+305N4CgGRPSi3rspgLzhnlkxy9BPdy+sH+D5fObxgxLE7w4KLnzAf7glIVbP5pyT1io7CfZhLAMeg8SAV4IWa8uN1i4Ab9bM8t38FaNMgDg2RW7Jn349gqmMwpzpzbCLgoqQZ1Vk5FUlQALMxB8m5D4toMZSrg5m2HIMcnPmYxDEZjdinRtnjhxKp/aVCRG1ZaOvJZ/6NKWR2jlts25E5O2jp04EZG0mtTZbXJM7gCYIamRfaxp4gAHHL4Yi6/xwR49W9MkIc6O/kE7CFwluz0pGSpN0QlfWN/skV4UEc0qYTinP4oy79NcVKsIWwVcf3mP85rvxvebM6w/LLH5t/xgxjYCC4E0X3TKGLRbXrzCxOi7fsp9SAAEdM9ENvLMqyzEDMVBJQ2SPpEnpCUUlmIYJylkCkzgYCuM27xWthFRtIfkSScBeyBh/Mn7N0DznTj8zBLLY2TOMR2hfvp9HntW7hz1u7TnBsCX5BsgIRGVv4ukcC0ZaNLIlFmYXXSyBvfN5/e6MNxv57A24lLCwX/zcK36YQAaZo6NO9I1JgN4uWbdqoF9cqgz7f++mM4DZWHwSzkXmdk8lvOb2ghQQqxkw9OVoOZkgeEi6qRjuwncvkyTkPbI9pEZFj3YsjC0gXuo8nntrSswndVDKj2pvIlx5XhTItw9ngHfSt96ANp8Pmw62gyFWSp/V6VJH4x/nl6nqa5xqmH1y7JYmd6UNspggHESSdcb1XbbjhDa8tpwVdKV/LpG9WgVIhzOkyYcJZNgJ716t3L6+cvzAy4kHGA31LjbLrQ3vdu1Ko27utipNSM1AXg7ImUW+mAQP8qm68AcBIUjbUKUIAt7x9clb5qqLY50qtoWkF6mnrtJmHxOh8ma8Pq7WiPdgN+HPXWhDlRsIYnfmzL8m3DUe9d+pGFmrvb//TGHg5piSZmIW0wA0P28g6sivLCezc6WRTHR0QJuPGGQJK3MqgdEAjgJ2+CWD/8c2hIAkAxgJioBPwO6GyBMLBN/rOOPYF9wGqdxGqdxGqfx78b4JCvV/aqFGVrO12Q+A3wwqM4H2GveKtWVx1zE5NuuwU4qx3SoQQfApgyFQMXqwyXv+FX8PfeITrSjKwcnPrfGE6ItbiKYaEBhxF0mh3eHAv/GKrFfq8CJ1kakzI5cV+qjm4gh2rxTrT84hX/GOoF8gXLcjo4gmfJZbNSgxg6TYpZGFrVr9WIJEBMCOhjQaHA9F42vDVgbrgaiBeLLbFLuYW3CvOV/x0R49xv2zoVN+PM//Q6/umWf3e5+phAyTIFFKQHhLCAOBQ0wIt6PNZNQNPKqKZVGnMSBmZ6r+qyzpUDl/QcD6ids14wigJ1uzFjOQSJgEGMIsxyRxHCBnJgI3DXlPcTNCTt2csrEnOgSglQNaUoSs8z+zciA8UAQQk3/uMynRa/bVPMcz1PxprapVDkjaeuDBmaOZ4eiamUKs1sgx7iYVKgSOxfbqPcR60wLScdNckBHlnbrcSRXKncaiKH0fG2agrDQwWhkISCowpL//Xy5w28/sJtV+tBwJF3u5Dwf8PIlmz/s+hr9XiovJ/mmgh41H90RPBlmUIcyeML8d8W0ZEoESkbCtyHEnBy5NkAi6OS1uoQzZCKhy56+u4JcTQk6OYv0yEM7IxuJq1QlOKXCpgUK4ai/PkYXUpUAAXf8AmgeCe5QyGadPHZkEvzeFVSlKu0ZGoD6MRu1MFs8h7+bzqBaZ8SG55N8rat96bRUuwIZ257P1/4Fv3h4xRreSOX7/FjHJ7mo0s6CokWoA9biZjOOFk074lwgpTFYNXQfhokzj02IzSSPdF5YjVkWkk0YAlDceB6NQo6hTVPUBuSLvIEigBqIuVfnC7xCgSfWHEY+Ek/aAJCqqDBlsvxQ5c92h9Lj4Ae4LNTda68LevVkCpRFDLllKYc5lAmTPH9/helcQiVi+Fgl3Hz5qPZ57+4vSh5sHVE3YnQw7+CDVcP6u79+AZLvfPOzB3y3ukB3P5PjsgpRUwS8tB27Nx5m7pFaWThHU2Bz4slZ+9sTNmQMpP3mZGWyFDjStR5eeqhp74AKBervSXNYp+HOwDGkSSbBiaTGHxxo4yZC/wSzcvoeyUwWG5t0MSNPZWM18rmfTrzZLCEbyWd5htsV5mhocSxTSKV/ngzUVCRVCZjYTh6FNuT+Z95o7EqIA1xSK8LcGsimBdGVeyw5Psb8vuZs1L612/I9lxnWYR6Pcle1Vx/lvpYN1G9vr2G/khuBgPYvnnAmG7TGebxfsTNB982Zyo9CHYHesEEJ5JxNNpFpUXqezW1VFq+Io6Sl0DIzPo+82OYA8WzZZwagvS3ynWmvlGK5LlOb02rP3As/L9dp2l+liNL+CaU32l+VzX1sZR7SnNSS0cpWisVAonuekIQhn0ax4Zys6FnC5PYTa0kCqnUxzZ+GqkdpT2VmPGey5hN1zHjefFGg7XplYEYg/KCf/2Mcn+SiWr/cw84j6soXvRwBhpIupLUNmnByfb7D05ZlD72peYLMVnB70n5imMejm9KtrU42/fMia3EbAlGx7DOHUvGEBgjnHiQa2djGYzJVKg9h6E1ZnAnFfUYkQznSiXtw/LOVxTF/Nuqou/UxFes68gS3sqrVSyYVJ5bE/ZppRaE9m1c9nA2420vJRQkk1TSZpL2u1XqBMBjYB9H/dqXPc3d7zpsFcbmpHx2aJ5kwLKF7Jt/acfB4lGsWEYusYOWO+k3JTOQHJhVSFzg9xchn+cFq/B31/KCrk5THkS9tmCftY8c2aupLOFiduK1NrL2UhcjdV5huxjmmbDLT5P65KZ9re04uUbLQtiQjhTYxeUgOyy8KIc6MnACT+7sUChcABogSvm7XbGuZq9ipbWb/PPD9l/W+bYSRDUMKRv2UaWA7xzyZuv0k1mvgRZMkjSdNEBG358+aSq2U6DMQzFPuZxe3Hv4utVpG/uwfv4NBwq/fMbKRnmq4vOlYlueHDhbVgzlCXbSqbCPsypWotqNdr2y+dFMJJHkWbD/pGdYJdiCV0Uz15lObP6Bshn54HH7GZJ04QY+KZh1wXdGORkd6jsPERtT0vADme9zty8IcWmA4j0g3IpupIkLeRPYWCKRuUdXEBzi58h5m5OuRiX++hVbdrNMluI0c82EiCyKgvyjVNEMA8nsd61bxRxD9duqpnsZpnMZpnMZp/IHGJ1mpjr1DsA5+tMXrOxLIJGUUztyIITD8+LiZoxdTAbcTQ3pBnsxYZAqwDNWplKA9lozkimE4j3j75gGbjv9w96tLjGdSCcxYNpAlMGZeICkOlbagzNL0hGTzi4RwLaVGb+GeCh1+motoDyRbKYG2q6Tev8kVaUm1JcmAlUrMlV4k92smcFCaVMm3Dd6vb2DOxKWptyCpAuNgMWS5hwVQx8LCbRPoS4ah3l5tsOka7Pd8foaLCvVKvmZTdu7zqwOsjdhtiwyFtsdJKCVQe7Jbn6cif/FAmge0YvThvcHYFZpwbJL2i8x4XOmFNiGK6YVdeoS1MLFHghf3oPqyh0lA/EbSUIZy74RlrgJRhkCVaTBIwmaNy4TRG1QiwRifFZMFioQ4eQ/3ZNXIwpvcGpjcg7lnmCVWkOvpi5SGInB4JehCG0CU2HsXOCrhTB0QJlKwaltgwHGJAq027Bmc1C2KVE6SJWjVWu7pVYGJKUDh+5y/G+XZ6m8SGgmr348V3n39DO07vqjDVcR4I39oJ/3bkWBCqfiVkQsxk/fQ53AqP9JzNJHVadU5QYg0R3nI90v5e/LlugNAvS5QKIUEgzxncFVrJyYVGobhE6KFPvPdc6C/kms8C9rrNh0bdOR7fDxPCOe5ZAaojqhnfHDDvga2Wb1g4A5lrgizgkbZfvL/GzmmSd83V+dmICy/Tcc94py4syCFyGnSbtBzPAla+DGPT3JRTYl7Y8EbtSUzTTgKfx6ixUGCn2lCFqg22dWkPDhq4TaI5CLDbz2pxtHfFP3XYtnBS7g0AMTnQ5lwKrZpsxJBlSIV6zTHsW0pyzqaCDMrDZlsAE6BjauzA4xfFGp/JhKM5wX+Vewwle9JEZIeIu9dTbRoAx3JLo7SJeTnnJQSPcF9bPTtVZazHIFISGK79vYX3+PFnDGjx36O86bDN4lZZHE7R3cjx9EkjNcy4XvLbkUrXqjd1pQ+1UiSaiIPe1fIMOQJlUz+oU16XQDA906h97TwMGunk3CsSkpKsomJQI1cw6daF4bxKuDsJUd9eG9x+DhHNQ2rVghTevOyUNnWq3SL2qD9W3gCecL4Uszeq7JApcGgfrCw2YVokQqEGcRqUuC3cOULFGoS0qEEhScDdC/EZvGsBN7ryP/0BtGXPmeGud2O+3bTFJbpucp/yy8W6017oOOJvEEhU8XSb6YkvystjfqJ0MlG5X67xPkDYf9K7rl5YPkMWK+cIephvyibIvD1rB+F5CZyr6w/jXVZDPLv2smGMy9041IkceC2jgnl7yiURSJWvDDlBJgjC1BDvAmR82z7siBnzSmfG259DLJhMGec9AQA1UORY/ml3PdnfJCzZa+OW+OhQrMYVJaW9raEG1QJHoU4Zydks2mYgRl54dfNYVNsD+cfEmyfMC6FgPRi8tzFMlcmK0SoybxhD3SS1JzGaZzGaZzGaZxGGZ9kpYpIReyfoclNhf6pxp2UFMu2Vyj4frOAlRBhhYEyeuVK5ZHqhBSLEwsNVLYtk43/ftfisG8QsxB/NHo85rFCvPCIWcITjDJ8ySQmsuTdXSBQ3u0mFFbg3mD+rjB+h/Nils4xTYW8kt8H4ApOc1Iju69kFqJfTAgfifgQMrmqPabXA8LEBcsvMukivergxLPWf5iDrga0z3L6O3u2Asy89sZge88sDHoTtNqZvXNwYrYxJK7gMmTtZ0mJVYGSmIpL5ZFD18GVhkKAbUI8OBykaoNNygSmzsJOjOGTKdfaCAnF5Bg0IQUBAOqI7YpbB2nvYPdGPWvTLBSGcr4Ecs7jQ1PcsnqjzNdkE7syZWnP6DTUutowgzJLn9yB0N7JMY7A4Tnh8FbunwkZiB5qNE+FKT0uUvGY7izHxAFoFgNCIIy9ZAvvCiltaoYR64QwEiolO5UqbTxjVjNJ4ADZhCTQfma05orFDjgi99QTf2WOIuSf++uo17N+IqkYCys5V9rBG4y5+reJvbTlmTeDQS9GLbYnDmufEISKaQG3ceocJL4rkprhogR0wzAJK/+b3afkZ/HjLVDoBOpMBeLNsK9G7bXlPcZFgj8vrlK4ayAIPxPe5L61bQBibhcBh++XBWm77tCtS4A8RdJnhkYJAMndgjippiftE/JAbMtrZiLz2b8kUCrB52ZMWHwn72EI2y/y8QY2RZHhHh0/41Nv8B/p+DQX1Y0DvBP9lkw4gwFsUleWm9le2b/ffXyG+b0sEp71qBk+dTtT4LaREJZRbenIW+251WeD5lwaSvC9Y3cdgCcBmZzjpUc1HyaMVg7cBsA9YBdLH9Um7WnR3ik8Va+4d5FvbACYvZdNAQHjORAz5Dlx9EE6ZhqGGTCIbCaee33YggHsxmgqDgxgdznLNTHsKItU/XanaTPjaJH+nhdOB6D6rMf5XCRM0aCX833RdBijxeVzxsrW6xkgEO+4KIxBszeI86iyglRH1kSCZThuW5xkpu4zoU3aA0pLz7B6Xug8TRY7/jEzPfm7yUuLCD+f0Dk9qeykmo0YdxkrTwhXHpR1vNtKz3eqmAFp5NyZngDpl0dX+n2mI1AsjlmgpJOwX7LJvBPouXko3zNWPOEtfiuT60s6kt4cwYVV0p5kbBJGYWl2OQyhK5sHN2HIZjjPdpMeO5j5q+lKZwE0C6hzQMWqRTYuMl4WnMkx5wVluEj6Hm5r4Aw0Fzg2nPqSP2u4jEf8hZwv68cSTJDaANo51XknT7DyvRIAVAV6NmM5V2Yk1LfFIYq/G/9s+9JmJl9yQQEcaVtzW0Tvx4llH0vgZDPY8MEom3aSQ0uQTVze9874vAKs+w7iOpY+NqhEogIA/U0EnjGtNnjDX1bOlV0Z1c/bIbez5LOrchxuN1EdNLLITr53dmXK7HOFrwNh+7ncB/OoiVi0cfzM5I1F3gCfJDU/zmH3BiYahNcT/rZJmL3e4p+//hoA0JgRf7d5CYA9OnMvx8+kz5O1nWvSqLepDywAxIsRlRACjEkgmZGHfc0Lu5v8/mUWFxqM+7pIKwD4rINNUvnK7GVnAUFIBskkRHnQ/VJE/pP+aH7QQyuJODm5AyjV9OR/dc9ZCpI1fqYOiGLvZjciNckh69P+UJ34YZFqYOwd+iwveXSIshl58/OPmFUjHiQWbndoMJNg75vZHiEZTrgBELsSteW6IlMYAaSKirduHYv2MssgJnIHRRQck5P4BULqXekrzYNKY0DSA86T2jDZxQ+E1ALNOd9DfnS4vuRNQIiEJ1+uX/SmBI5jojF1vAFRyzgHrf6btdHj9TMhx+TNRG8wPJeq5HzgXuWGdxPRAUnQheYpsc+rCv+N6iuNJ42/83OOG6xFBkFPhCQQyPjaA4PRfrEZjvWKNnMLAvcLm5XcLyNhPJMqtrEIBGjgyjg5j+EYRYhV2QzSlztANpfh2xlXk/lSPxr9Xt3zyHaXk8q/v5vpMermZO5RnffoH/lBmX1vddHIIdxWfnlsy/3SPBi47pi4VCrL8l2Mx1HVOVwe64cxqVzTJJA7G5DwC0ws0j7kZDOoul/ZyJi9RfVe9L6TvvS45M1IL/cIzUNBwu5qUFWiH6cmLqaXczXV0ubK2QE+m9UQH4taMNblmDSN6w3/4dn5DlvxAki3M5iPYvfaHXM2psSsH/s49VRP4zRO4zRO4zT+QOOTrFT9eYCZce5n7jPZZx3O5x2+nN0BAN73F/i7v/wJAMANhO669KyqtVEoZJo2E2cRZjlqy6NuPGrp0R4OtfZ2EIghqMxMrSZMT0ogm1RgH73RvmkciKtA2f2G3pYeBEEhZPIFtgF4N513+P5cPneK1WVnlI0pu2Erfdeq9OPcI79p88iwqsLLEwarORj+u9wPrI1ie7EC7LVUdtHgm/tL7SsDQCupNO/W55wzKo5K9Z0rDi2TvlR4Hn+fAS3pOGERkVzp+fl5qfSSTZOKk1NRsrE/bZ1mw2IeAEpaZbIqX96jSqiWgybO1O2I9U7cuYZy8mNvYZrCRg3bCl5MC2jnQK5kudqD0R1797r0kSkQG4TIufLXI2aXDJ30hwopEEZJfYnOaDLKuCAM5zhyBtJ7qS7MS0pcpeSKpbsB4pf8/pYS4qZU8jS5daZSLbcDFu+Dhmh318SwI8COYAmaKWsGo73qcclmAbkn7BcR9kYQpERIt3xO5x8Yro5SLR6el+qIIkuJFOYllIQfB7WFdFXA8GGOs69zqwLo5R72Z/Ho/g8XXqs725kjaQjAkpj82dl6L9Yc2pANGfy8wPSZTaxQ7oQDYSfnMdtp5gp3KtkBmPdgN/zL1QTCZ1tRmRfayG5UWUHgDcPeAOyeUI2mSKsmI0j/VjNfq4RqW5je5fk5/i+hmFUgMpphREXx8M0lmo/82bMex0zgVCQ7bEiTSgLSj3h8kouq3VmYYEFriyB2df5sxEXT4W+3rwEA/8c3P9XfD/NUiC0NQ62j9BBiPSF4uMjkAJmAut7+/+y9R5MkS5Im9hlz92BJiz3SdGZ3dgQzs7jjf+OEE27AARBgF4OZ6Wn2WNHMjMggTozhoGpqHk+AC6QPW11hIi1d9TLK08OJmemnH0FkOzB/dJV4YjNUG2GbIlTLCDwR2zYiJYVYenJRIZaXqtgUMsCgFrG6/0w14itrgGz6wH/P8Nf0Z303AllJLxajETvD+SSZLYA2wfJiEE4W3akubMPLXFU0E2TCT7Ygdvz3oNAwtT93QXrFH/79HrlLWL88AgA6F8S1anhuYZ4sWoZJ/VU608GJJpah6TSLZivXWHl1Hr3nqtsSXKr+pgwbFhkNTIYqZKGkaMNRGn5rL85LTUN2hpnh99FrdAwFL5YT+UoDSL2l8wtMtLrrZSOR1hMRtsq5mIzIEp3ubV0k/IokEiWhxS4DhgOtLsoQy6fYSVJKEk+uLfnStk/ca38G+td8zOtZP1gD8aXHfsUQ8iLVZ/VkSNtZLt2hkoqSqxKR5jnDLzXGmyLPgJDXkEELVFn0dPW9zg1o41Ke8aQQH+k8mgeDjhevrIDhqyzEM3eoMpS0SPCrKGlO7klX+8TXXlop9t/XaAcIbDzeVMmV2dOGpny39q2T71ZISwI97/LZRqXcs9JrLRC77Wdh4yPDpDN9q+imG4iDld/Q8ygWmOocdjZDlbyQjpyPMYOrc5MBnWUTA0B4H2aic1LlGLn2qf01cQbK8ZGUFA/zoPrEvtRlIxQXdYNedNORdf1uN4vNnMfpeYXptsr5yoKd8Pkvqhf49zIu4zIu4zIu4y80vshKNS4jQXzzLUVv8W8/vMbu5QzCKw5Cue7eFO+6SzB52oRz2cyoBZJVJsMzKWL5vRV4xr+ZYNtQDRKSkl1lsIn+faG8RyXidT2ym1PJeTUJmquXOFiEQhrhXWnZgaoIJPZddSbB907gWeVrlUN5pPydlxFKZ5HGqMEQkQHAdM0SBN7FTrep7pJb2n2W6hdJwXPVlr2GKsSqdYDpIobfUQmdnhRaKlrRZtqBFxgtdUlg6Oh0TfRJiiQeM8KgMBI9GSuIzGIRq/nALKsWLsEtPfzJyc+0KexQQz/nIOvy3wGg33Zo3jmoqyKVyRjf0wnnpprr27VHeG7kxGLUeHFNJVBnA/4UDPIDy0tcRi7IyX2q9ygqqOsJqxWVPadDizwWyZWBnlxkS6QAACAASURBVLSY3KcXHigh68V96Ibvzay6UL4+p/pmxNW6x+KebKsUSEYGAENoYZ+MMF/9qlYbcZmh3pdzBGJTvWiTy2fG9WkZaxmoKnSIxEYqMxeiAjmqUIk94y153uZFls+JH7GhZKfCgIYC/DWz8z85MXRwe5yZmLiDEua4eOyWZynV96dIRsys6ixV23StMDEKFLtzGZcOs2q0ZQi4tJtmBKe5o5bb6zOSYVjmMz9icnCr0HYsHt6p3m+4BN3WlpL+0Mg11f5ctjSHtPXEc8esLVD+bDzgS5UZFMabKOcFDShuweTJQA2aAjhQqul6vqXC99eUcGS/ogdreuzQfrCIw+df532Ri+ovfvsRdtViPzbCTJsODVabAU97mhVSUMgF1tCACiX2KMPfJehbWmFMVhLDhaTQ3vdE4wegfugkfDmsMvxLmpzb1QRrk8CAwVsJL4bXwKSrXhQQg+vUZMRNhFvWSZ69/BFHIwu/9mS4XvsmNdlmSg7qZOU9pmSU0hNC7dFGBdUlJO4nFucZgCaV1NSealxHaO7fECxqAYYn9cEglb6sTcjrksINmD91NdWkAQ6sGWweKei49ORWr49oLP27/WGBwPKaxfeW4t8YHiMoq3yXjLSOtXe6t5XdvApn2l/rIvzEB2mTTCRu5ZGSJhkCj7L46mcLf52kr2cenEy0i69O2Czo+TiODWITpF/c2Chh7J8OK3SLCadN0ZBo2TyQ3IAXDZeQe4sjb7zUYOrnNMhBiO+v+76ViXz82sN9XRnu47GRTU25BwBtyLZhhRNLXqxNGPsisKS+Z3E2MkM188+LiElkXDRhW3HEUdL/DIo2jvPN0NxIfa7nnEP2aQHZuBHcW3XItPjys7PVzDrle3RV4X23rwkt834nXdcsm8hmS/Bv5POYbmqkm2WLTIFCm6rfnq7rAp7aRMxu/ppTk2vyjwbZiJZO0WOVVulwnng0HzqcL3ypQWUbRwDHuliW/z4tNNBGmJ9auXaWHcSyov5n4g1DNhCXqexYIjVjZs+d48qmKJuM7r3B8JK/wGwBR6T2jJqnRck1qNc7mwwkhfCWbkhzpHNUF0P9y7iMy7iMy7iMyyjji6xUf3q4hu47JK+F/euWHrfLHgtbGagnTTu95XrE6UQWKnGZKSpthjlu7gm3vFkMeL/dwP2OtoF6xnaLDYQAk6LGFDXBsDyKA0xOivSWXIlkrxFm0V1mFcREQumMUIhKx+pRS/+wQtbZkGEDAKiTJVMDW48pRJkZi9dsPBmpF9bgqAS6CUsOMC9MPZOFMDUlB2ydMJRzO7PL2VsxbW9Y+9e/ZPblVRQz+X6joE9GKoP+1CA0rFecDJqHUrkzFCZldyVNCHmsfLc2Yf2K7tPtssf3P5CZsG4i5afy9ddNhGGYN/2whJmA6VXgazJCsWm5SgC8Er2fSkB4yeiFSRJxd7+qjlEAsZ53PaEjzx/X0AcjPChy5CqwvK7axURwoXhMeyUh6NnkMy1jWHGFDjIcCcEgFreoXDWy0BDiU8l3HRcMzd8O2FxRadMPDt5rYteCKsniY6wfHZlNgBAcFTPE1aGbmT8sE1SqzlfZoOb0MjlOUJZQ0Yb2qcaqnV4ngVUBrhb5c8RenlXQTSayEv9suOf/bgmalfD0jp7D8rnxNiN8w/F0g0HDAQl0TrWaTk5huuXfHSrcawbyYC66Y2tV9VNekza8oE7EOOfb4mda5WKewvNSWJB5Bt3QBHUycg/1pGDHcv1nsPaTgXowZ8xaYewqMnWR7NWZ/jl1Cfqk5d/ZY9XP+k01Pmm3Cn4F5Cs+6RkJDQDswaDZze7nsny3WUWeSJEgHtAnBbev5MrPeXyRi2rcNchTg+Z+wLKju3jsG7x9uKb+FwBz0Lj5O5ox7pY9wj/ShPxwXMLqhJcr+rtSGQOror/719dYvDVVlmJRDaRtFqZbjGTkbxjytS6IDCM7BdNFtHxefrLwbLqgG5qsDUOXxiSEqaZ9zBcXt68OPP6NB7hXYY6a3HrE2UVJ+gwA5NL7Gw30zmLJPTN7qtZssSEqvLTInq0k4pSAdGFz2nxmZFGM3+1A59kxPJ52VuBCvyLoTUzLQ4dULAbjLNS7IajW7qsrToGa1KDJCo3PcfnihFcbdmgaOulXmo+OJufigvW2k71JXEXYX/VY8KrXf1jClkWUbecKfD18FbG5poXom+sdnkdaOHd9h350GEvg+s7A8YSzBD0jhdkcbZY3MrtEkh5QTx82ifmGOhmRiVBKgYJdc0tAZSwZxlUqoz+1UH19RiRMfFLVBcvR/TRvaIa2NsJzuyNwP1yepZsIza5PtgdiV++n9rVtHVY19xOcaCTPS0TtMTc0cRdWbDYQuc3hV1V2ptgwokzCWUPCr4vbVOl5to+m9jIXdUFxe4IYS8h9+6QQeWHrv4rQ9yM6dmLybzdiN5gc4I5Z5ELjfZXK6FA3O9nQe16g0Nhlse3MwZHUpCTXqNkqlOr1DesE6PpcnTFrj8QID7zIhiuITaYZVO29Mlu5uKFB1QSiDJy5vkFlSsICgMEiLwMsh79TJiv9yJ4qZD/eZIRNBvqy0ao9fXugNpRYK86MLLSvvdzKjK7XIayA+FewIv0VfIX/H6NNQJtwtRoQ+QXwxwaqp7BmgHZwxdHnn25+xJuWGiu7sMQfji/w05HYCSfv8PBvVPUsPmjaffHiELuqQUxdgubFME3k8aUNN/dzfWAX1wNaF3Bkb1Q/WKmmtU7IWSFx2ZliTQxxe11fHMV+peVhfrZnMUvZZiESqVx3h2GZJbJNvW+xfKsrGemqLmZ0kHPaf+WgKCJSlZ7fpMVKLQ9G/s10TfrE8u/cvpKMDLsmletYerjAz7R+RTYxkxLMJxZz0mh+RQvp1zfPOEx0kMftSmLUwlWiCL0yyd95rG5pcbxZ9vjpww1V3gDsUK3fCqFjuuPJ5H7AxL30f/nuDcBVjjnRMyGYRK5kMM0LhSAKVskkqqLGYkOz+v36hJuul0v/x8c7HH+iDC19Il1wKDKORZBEEgAIfXWjKlU1QJu9dF187DTSdYQrxJ+s0D/zypNAQfZl4h3NLMIwIDlX79nPkoxEa2kyaVXLRssrmOfycAJhkdB+MvL7xpfFlYmqLroedO2k3zevzpf5bIOTzexZ1Zily8z+LYouk/7SfTDQPyxl0u+O5EZVjnf4JVAYg9T35eelrRUcPRNV2oJUfYCbnYaOOCc1lWcpKkxMqCsIkmiXDwapoC/c72/f8uYq1gVdz3zGp1v2AeaNo302shkppD/DxKISeQgAAwDz+wXaB75tY8bEdpV+nWGH8m5RxJ9hLknYJHFXUxFINotbl5wbaLNkD+XYtDGabvhDmQhUuT6in+34K/gKl3EZl3EZl3EZ/22ML7JS3dydYJYR+1OLkVmqyIB90cPwbtr/eYX9W6oG/if8Hf7D/UcAwD+/fwNrI769psq1sx7bN2xagAUJ53kXqHSWSrJdTdVpxyvAAokrYW0irq+p9zZMDsdTK7AbgoYukBT3YN1y1nhgvCQ5IGDWY5oUFBc3bqfEADws81nFMqfNp3UEuMppOY1E/IRX+Swn0h1qEkV2WXbuwpgu/rkuC9M2G43hNWfKfmcQF0DDO1e/wlnGZlpwMg64Mi2b/+sglXvqCWqXvuZYe3DBJeQmC8P6+4cbBIY0U2+BrlYT61/v8GZDxrfORPQM5//h96/hHm1lqk4Vks6GgryXr6kNcNotELk3OUf2iP04q+STEtjyDLIH79ILopCB0xNBxqenBb73WiQq4d5L1Sc9seJ4pLKwz1NQwKRhDxWVELP3Jkt/GADypBEMy7oGBc3GJFpnQlJ4pF5XNme00jpIDcOzzey7F+8HTYERpReocmX1qqjQPtaWSf8mybPU7oxct9iQP7Hkk84M3bVnCLnMZrn+DKFCrsnRNU6zz0kAfSDExg6lAgWO3xRmOvknF+N5FSFMZu3r740Nw5pnebD1d4cl5LupADHVKOYedECQJKYp2ba+3tuDg93rmnyj85nhy8ioibqaqNV0qNB98bpWLTHiC29AqYxjQSWeHbqjkmsQulnaTKxyoOzodkngQK/OkLDU1Oo0bKJUtMQNKPeMjE7KtYotX8tZsPvnOr7IRbWxEcZGnE4tNEeR5aQQJov4yD1Vr0QLeDx2+N/+/LcAgPWfNcIK+Nc71ldeBYFZ3FcnLJqIaaqBwKVvmpISeQqiAmyWeDetEw4M94bBcRQcfdRuvCwMpo2EQBWoReeZTWGe9TXZ7Lr8KFZYRfuiyeMJw9Xel32yVbIwcV/muk7eBUYzPfWwHL/QoZvBVVuLtKgTo2oiEpOd7M6ehVirULWAxf2HzonhydI/sxlrDv2+X53wbkubnTGR81IZ5G5VpSYIShaYu5sDts80w5n3VuBB97rHOFk8DWzsPzToj+xWNJIGVmCvCAyv6PjrX+/Q6YTtI9n6qMOMGILaixIYtECww3wxoHvheeK1BwVVFs4FYJmQ1T7SNTn+uogLFTT3UBMcMGohwUFlWkxBz7TudV1sZs+BOWjRrKaOdLUlDcna6rakFDAOrnINZt/RnFRtAeRzR67YQe6fSgSXBpaUYdBwHKPWPNMCefpVWWUhfcjpOsnxtVdniS0CsYLgc90rIf7MIV89QTYu2bBkprT7h9rjU4n6r/u/qcctz2qzVVCz9sTc9EcHQDPEayyFlpfElmTruyXSmLK5CoBnyDeuE1AkXi7BuCQbmTgYmV+arSY7wBJudVIzvWwSON9oIA2m8gSWAYvZRjwELWTHw7s1zIHvxU6h2daA8eEuz9ouldBU9LDleSdL0Pq9sq5EwebBSFtKTz8je11VOVL7SdNm86+AqHSBfy/jMi7jMi7jMv5C44usVAdvYbxF23kMPW3F0tFCTYockgCkpYLhqqH7PxdonwrrMMNMCo4Zp2HRCL3euwb+hCptSQqGjRp879D8RNu0UIyjmcwTvEEuZg+jAdqI5RXnjHoDzXufNBPNA0RUKlUrsTqr4Fp5VasKDuwGKmyVZoSIIgmwx0qNp2DmWWVzqp6jUDhj+JkJUCX/tcnILlUv3W0DU8IH7mZQVrZothWW1r4eT0WqiDLDluE+wHLO7RCsVJ9go3NhMrZRkIGmCVh1k+TjftquYX9HJaG/TpIvGSaD1Ftseeeec3W3soOG7SvhJv/2hAVXcykrGACGYbqobTUcVxXK9lcJdq/PItKEHa4J9i7ZvN0nLfCpmSoBJiyB0y9qaAC8gi6SiDZSPCafh7UJTcPGE0/UjsihVpfiuqMhrmDoyIe65ei9zgX0E92Mvm+qsB/0/BRyTLhOyJrZpxNBs9PVjAlbWgJNgmrOz6PArtEB/a8rnK23ViBNqFpxxgVV+4GvlQpKmOQqUkB8nlWkpYrKFmL2rhIRZIpBRTLE5AU4j7XJ8s63j9UwQWLpymypcIa4FPKRVOvl85xnCxCJyVSuGZmnsDOa7nXNSFYGwWW0d/RhrRPiuxIiC5hQwzFik+Fv+S8bX9UFewe1CLBtfWaK9C6MluRvxa1r0mJQY0+AXyv0ryv6VeYLM9S5ITWZPNN1nYzKvdUgcqPdMgoXastExcq8hgbG10EkRmbCGRHtcx5f5KLqTIQxEce+paQXAO5mgLVJHj5/bOTFdEcKeAYKTDl70YpWEqAFwyWAe3fKK/iP9EKYg65uIncTtM4wDLMZkzEeaTa11yO0SThxn0PbNEupMaRNLSbUJhPNHgSZnYU09zUnMZuqe0uGIJ3Sw9HD/HMznVhWZwkcyErOXxWJRDkvW69HMfMvbFq4jHRLL/dqPQozVf/okGyVRaRZtqL2QCgvLgAkYPsj63lsRssJLfqWLBcLhN90AXcb6nFOweLT22tJ7rH7ytxdfaeRf6T7Ml1lTN94JJ6R01R1gJRtmYCv6Pd1bRCj/G9ebDFFg6eeoGiYDM/f06yDWEuaHzty/iktXFshdTMoTHcRYL3fcWllklG5QpzjPVn+lWvsX/na5+Tnt/buA8ay6eD2QNl0KFsnwnhwlA/Ln7u5Oskx96cOw37m4p5wnoDE0LkalbSE/SpTr61MvG2eyTYA/bGRRapIWwDg+A2lH+mZPKY8m1lXuZFmy8LCWp8nt1CaiqrG85xrCtBCUTYqKgOmr+zxsKyL7+KtgTvgbGM01wn7TX0+VQLUPKmlfGeX4TfVRhR6fr507cpGN2ug/cj3zgKe5S/udkQYLfwPK75WGmrWb806o2TuTvcRih3KjE3i/KU3HjlXDgZmaVaU6JSguP/ffdKSoxuWwOHXASj686AEGtappuCo+5GOWW6+yjDscqbZxKy0ovSM0TsPoPebRKEXRdN7AoaXQBRHrs93fJGL6n6/gI4d4sGiveWIK5MwnBrpe6rByA0/fpvhr+hJds8GejpPupe+ps44s2A7Vcuy7DLyC1qxFqsRXVMnxqePG/k30WiEyUjFpQCEQ4n0ANQyQhdh+K5Bs+NNwG2SCtnt6PdKqsSM4l6qBylqVX3R52lwKoPSccr6omtMkyygpk5+MrF0CWrUte/yZsA9h3c3JuLHj8ShV9cJ7Sd9vjMt16A97z8hVrmAXXtsljQDPx87KJvq9fAGD890Y/xkoRdBKjrvHAzHtMWWEjkANhDwCmnPmxhfta7+JgJdRMeL9mYx4JoX9Jg1vnt3J4uUufJYruhnp2MH/SMdz57UmRRpeuPlu8SjgXvVo+MKsW8aeDYcsVsDf8+brqOG29fjqMEIaQQuQQ0Gls/xNDQiuXILj9QYOCa6NS5gZLLcFBtZvNqFhw9GDCuUmjF9PPVrpZ+WaIMIoCa1gJ61jHrPzuLKTgZmwCxhJtdN6iKz4UHRUVetIukkK1lo/v96VrgXCVaxIzTjeXUpZKEmAzez5z3UStJM+JmJRv0u9DxWy0Hb181nrmAR4oJ5AOW757pBEESpSFtM9e3NJsumBX9aws3QjLBOUiE3O/Lplg3xqCWQXqks70EcDGnDZ4teQYjS0aL5aGvBsAcm3heefhVw+/UOkVGn508rqFiNVuSav+2gumrR6R6NxF8iKuJazKp1+Vme3ZdFJhIZ91tPr8n+NfWzh+ozHZee6mVcxmVcxmVcxl9ofJGVqnrbQXUd1CpJyDSSQg6KzMpB1UHp92VdnVHsQWG6yWcZh8V+LUVKtinwYWqz0PDDChJ2HVqDp2MrWZx6rKHB2SUsN9VVuj82khOZm4TkDRL3ksxJY3pRGKGALqYQDmeZlfEqQJ+KMT4JwUU0HhTmlovz6hYateo+s4hjCLm0gWLtN7lng7BMAKfi3FydsGk5Z9R6xBf0jx5+/4YMzBmam1dzplfwL2spYpYBK64CGxvx6UOxdlIwy/q5m6sTAl+DZTeR2xX3BmPQGH5RMNiKJqjBwByMVCGmV+JE09wNeHF9gOO+7ONpgd2RYOPh7YrDmPmQtxP2n1b1mGXjvsjEWr1ls/omomXB/fXXPZ77Ts657bzkzdr7HoFdmExJDimowSKKkUgeHBAprxcAdBcl/ahtA7zPAkXHpEVCVmQ2ADC2LWKoFW3XeKRVeaY1/GBrfz5XCC+6PGPZ1vYIQO2F0lfPbYK/UbD8DJ7eKIxfVe2E2VX7yrCujFkVIQzl7H4WYJ20BNeX1BXp3eXZ58zMUD/SdSztDunjgSrOaVOrqrmVohmB5rmmMuVZtqiaWX4iKWICJyv/Thj5bG1a2iLZ1Io/uRlzXwG5wVnro2UWuD1y5V3dTWH5WYreSAsAGRRQXr5b0FWqlZTMDwCZWpTc29WrI6Zg0XMrCn6GtOnKvNaBAjbKOxN+zsvQkHzYOFMNZJ3hueduDnRsud5/ReXdF7mohjsPvTA0uRYLPc/JMAXKWSWBJvSkhDxRkhsKvGVGJVDndJeQoMRGjCwA+YFqKgTjB4t8tGITl5YJq6+psbHuRnQ2oPf0VB4fllUXaBTBjeUcr2IlC4y6urx01Pcok2TqHRzD3P6FRU5KvHrxbOuCOYNn7KTO4OC5S02yrI8sSE1UQnZILRDWwHpNv2/wFnuGNH/aXmH8E2FN7VDp9QAwvYgyyTT/3Q63JuL5QItKTgqHQ1c/XFQzR4P2D056PU9tV3WYLckUSj8qn8wZLCcayjYBXstLHf7uhCtewJXKOAwtTix3Un9eVElES5N8gcD0UyMLQ+wywg0/A4tQw+gB3F0f8WpFcPjH0wqdC1g1tBqkrLBq6c/bwwKKNwxTAmAz98OArguiM/SOZFYl5N6YJDaW02QxHRqZXEcFuI/sDnWoxLB4MtDrSfgE+2khz04o8Gu5jkrVZy6pupnqWR50Vfvucq0mhcXbWU94k4RUY7cGanZ8Pc76/w3qQprpPWyYANM+Vcet1AB2AtypLqZls6ZjXQCzY8g31s/MYWJ3AspJJ1cXXz2x9+//C1FpuslIr3nTuBox9A0Su2mpaFB2NCrx95q3g/hwZsZrSLZ6MgOAO5hqYbgkuFr04q9rAzLH6mOu24h4nPmKu4ScSutAI82cx6CqRtZ7Q8Q9JuqZU23PaK+oVw1aOGNTE2f0VDcgpJmuG27bV2LbdF377Nko2IM+41GonUb6K4h++/y/wWVcxmVcxmVcxn8j44usVH/57SfYVYvvP9whbZl1uyejeTEc9/oMEixOIMkQIzHx5+I81DeBnV0qlCsJMxwwDgCYHHRUiNe09bv7aocd57genzt0q0lII9AZqbAJLTf3ixG6nTEsbYZiFun97QE5K/E1fnOzhzNMtNIR329v5O/TrcXx/YqvgRGv0vaB5C5F/O1Xs122zVABQpJSse7+fZOFzQrQ7vf9+zu6jgeNtqSCJJYV/Ceiga6XI8ZrdowyEcvG42ioFJmOrbj/dB+1VBfDq4T+6yjQ/NxH1zxYpKbmgCoNMf1uHoxsJ/VIxA/Zdf9pgeclVcWUC6nEEGOeAqSMIlIW+xonAOtfV7RhCvRqtTZgikaud8oK32+JrOVshNEJzwOHlGeF3a6wapTIZPK1RwpKKpGu8ZiK4f1kkPZOqvA4C1zwoz0PZDdZ5CVhNZOgtAlxRlRyTZA/56iAwZzBmPGGnlvdRKQnekDc3rAvcyWsFHTHDJQMM7zgYzYZDTNf4zITslMY7XGWZqMqLGiPdB+aZz58W2FotweafYbiamzaKDEwSLa6+DRb0DNcoH89I+cprk7Lz2aB5eJTXSpoBwRmetu1R+aqsH9ek682o19xlRBsrVT1oGo6zyykPLkMXQxdEqAnLRV5WGSBwAH6ziW1xqksJaEyCUaMbDTa60HQjNPHlTC9ixd1IXzFZRK/bz9Y5N7C7ioDvVx/v4k1zUqTOU0hLmUzq/gd/7kwflU1eFERcOzxbAaFsM7nygMP5BkB7XMdX+Si6qNBjgZaJ6D0Qw05wEg4eKoQj6rPNLH95rZ8y8rca3YaoctIbZEqVCmCPamqKTMZeuXxt6/JuXrbL6qxdUeaMj8yhGQTumuaFYI38M/NGYRUUjzcwgvLFgBCNAjcnztODV6vacIfo8Wbqz1aFo+O0eITT/jbT2u4P9Ik6dcsiyjSAZsR7/kcJw33YGvvaxaknA2gdMb+O+p72oNGV66xyyKRSC1JZgrUak3EifG8mDQeD0uCLvl64YauwXADGNb3GpURRovuGzrGNFr4J3am2ijkNkHxwtQuPd1vAMOmgfpAn0uWHWF4cxJtkonKfnSIiyTsUXF9AXC8jrj6ai8m+jevT/iba7qfQ7R4GFZyLxbOy73QKktI+WlyOJxasTWc9k1157qpffWwJ61K5N7g9uBgivRGAVgluCu6Pm3nxY6xDJnID1o2P1nXzZ+7GuFcFDa6tVEcd9LRwRy1tAim+7qzSEcnvdbU0L2tkW6zVkqXMb5IdSMzzOwGORRb/t1MDwlUnaoZFJotSVsAWlTLBlB72iD4Dff4FvX4FFxPf3Z7MonPqvQCceYElFwNOyCrPT6Hno43vi7N2Ex2fwDCcyPhDHJdClO5y2JZmk8WWSvk0iPWVQOqQt0gZJPRPM2ZuxVKzS4Bi4iGte8paqSy0EUtvfkcFAKstLbMqYZtSEReLou7FsWD3lsKg+d2Su5iZSUrnDO7larE/YQZ25ekVWVDFVazyLjHCvdC0WdqOAaoXz9z7Ppcxxe5qKasELMif93ykJsEmAxT/FtTrdJUysi67NJox6VmQvbuI0+Y3O+QhTkoyRUs2kQA0F3Aty+22I80se9PLVruYcWkMe462FXh+meMrDczluzkygKwWI/49T3F063diA8nmnGehxbD5MTecNVOMqlfNz2sTqLLbHTAqaXj70yV3tg95ax6pv3r1wPuNjSr7w8L5CcLS5LQs3zGcBWBg4MZi/i7JrmkZcT65VHOqRgzAMCn3VpMBp7fbeC2Bs1vOP90c8Ljri5Sviy2XkFf+TNhe0nLyB3FvmUmng2TqRW+zkhcbbmlB7yB5eou7B0cWx8mm6FHTTpTHqf/nirr+5sjxmBkof6b6wesONzy37cvpJL89c0jvl1u8WGke/P9/kYkCwBVFVIs2YwSVho+dhUpyeeL4FwuoSL19EsPFIA8L3kwgE3IxTykAww/j8Yr8YptXURjI16s6Xr7aPD26aqeo62EIeUVsuV7O9TFNrbELxD5BCDn6K/j2QZTJUURZ2CuQVbUvwOAXBels6oyAadvsvRezTBDEBRw/LoaiZihLpaxmxFlDPXySvqMOwKBJSnjDSVKFcKVHutmOi4IyRDi0tZA8dSpQp0n4iJRFmqpTkcNNVT9pvKqEq9slh4ldK1amyfNnsFZjlmsN806wDVB9KhxMmKlilirYGWojy3pMGNdtMMyn1WS+cUoPcA8GEx3Ce6WnmN/aGqDMENIVskq2GO9PslVqZD2CvpUCxJ3VGjIJh1+PSMwcbrPPPM1rH9GRvtMx6WnehmXcRmXcRmX8RcaX2Sl+rBdur6RTAAAIABJREFUQ08dGdfPrLb0LLVjbiCfFYuVQdVLXKSaUjEoYTwiA+mboe7gDg7jK2aBRlWh4NuIXd/hxLtYpSoTsD81VF2IXiVLr8S5CG2SyCKcjWLf9zCsMMZSlTnkrLBa0I7T6SSfK1Wq5pNsTMRXK2pU+ZcGjx9fAACMB6IG0g1VzP/52x/l2vzXw4LMCMhLHsObSt9Xy0BQJcNjMSoxe1cArhaE2a3dhCkZNIxHPpkE86+V4Tv+fY81myKcJlfD2A8O4D6mWXmkoDGdZpY53B8qcgCB5rdaYDR/k6U/6Xct2cSV1nRQYvQBm6FPBiOb6Hevj3i5pPN/2K6R3nW4/zuCfE/B4e2JqrtVM+Fvb5752ke8G64ERZiCxchV7PHQIcfaK80ZQF+U/jXvVLuENBqoktHaq/OWhM0Y+Brkp0Yq3LhKQAYaZn6HycBzWWVOWpJLVt2EV6sDGm6SffAbNCUZqYvIXiGZUvUo5Ka+F/4XhJHmpGA/utoPNbU60oOGmSrD1a+zGK4DfI/K8zPri8e2sqjDi0Ts+dI7BpB18bWkf1jyScMa0ju2g5rBxPTfw1Vtz8Tiq2JBsrpSwTWzSrXN1dIRVLVLL9bUe6E8oTs117VCvEh8TQrHwuva/x/UmbPYdJ0J8QFJYxq2Gyx9bumZD6jYsMuwW24JMMomjPZVBuYFYFRQvyDUadl59IXdftJIXYLfz6KGyve0EFvVDCB2ut5fT/aJ9F34v5VAiVNl+Sdb2dyZ+9niMmeqrOhzH1/kohr2DXTgt4YfPHPQkqYBEIxRIA0VZ/IPRX3MMgnH+yBWcQCggoZ6ZNnCjOTiVxmWA7Nv1j1eLI947mgReb/dwHNvbrEkaUOBgxeNh+deyRQsht7CLWj22HSjxJSdvBMNZYwaWmfRPPbewZf+qm/QmoClLTIOjcBvnw9GNGvHb4G8jPjPv/2Bfney+Of/41cAyLc125qyYQ4a6SUdzzZRJgEAmCYjPcPfvHzAVUNv3cpO8Mngf/0zH/O/rATymu4jMBrsE0tqThaa4fDm1UmOPXxcABro7gmSVSpjKBq7ie6nYzKVnhT19QBgE4AdXbemSEt40rG9gko1fJnISPR9+k9LTHtaHNPrEb/8h3c4cvD5D/sbkca8XBwECp6Sxf/+3S8QhhInomG5/7lYTkhJSf/SRyXQfrsesezoc6O3OJ1slV00Gfmu5ItlkqTw8ZWeyVBMRnczYNnRuexzB8/QZHjlsbym67ZqJmyHBR72BLEPD4u6opgMM1QIvMCvAFsH7llrXeLX6l6wnq/m+LHiodxGkZfpk4Ga+cOmpkKE6eWE5ZrOfegbpJNFe0V/Hw8tAiecuK0+Swgy40w3fQDMUPqkCv5qJklJlO4CAGkVyZKveGO3M4JgBuzWznqbNQZR+woZ25OmjcCqLsASA8e2n0Uf7HYzD98WAl37dUK893Ad92KzwnQqKz9F8olD1tJjKkHkD7b2sBcEVZfj60md+SLH1xO+vaNN37vHK9jf0y9PjrTHUmgEjcxe6NolSr7h75KbXJ21fN0IkeRwdu+b2vqae4lDMRRdyJtZIdl89nx9ruMC/17GZVzGZVzGZfyFxhdZqaouEuFnMGLckNpMpt68pQtzY4WkquHAqKGOBomhM4xGki1Sm6APrrrDrDKGF7RVXd+dsOaK4eXyiI+nFd5/vJbjtyvaqvenBmkwsHcVri2JIUPfIHkt2YgZELLT86mTEPQUFbTJOPGuO6dKZNGrHis3YWDcy+okJCZnI/Q9nWPsLe5ePeO7Hck/nj5uJD9V5XNoK76axAVqsxiwdFVSMwYrULNWWf68MhP+x3/5B3S/p/P36yyZrACwuT/iqlRYQ4uena+GQwPNVaZSQF7VqjglJa5V4sPM28bhVUTmykM9W4H2VaTjiMtOP2Myu0Qw3Y/MKF4nqG+ouvvliy12fQfHoQhv1nuBTzvjMfFB/q+HNwTll8qvjYhcXaRG0TmXotAlGEYhUlJ4ekdwsj4aCk+I9ZzTU/WDNtOM+LOMwvpUTcTLq4Nc87tljw8dYfbPn1bCEs6ZmJxiRGSyVJJmZ8jrtoRJLwAUQkxUMPxMFM9kcVuaVWzmpKEUtRMAAK5SWrPNZwSn1FZDDXiNE8u93LOBshnTRFTsOVRYoMRCnEsN0H0qUhYgdkxGukV1CgMn0xSP7Z6vhRiEcPXHIy5TJYopyLyhQoWuY5MR2CierkmdN+xAKTUF5s0aZ65sxcs5LejaF5QpeQ0UoxaboHUWFzj10KB9Luzc6qCmAqEGBQJPfM4AEF54uDbg++/vAQDLPzRyHqFU5uX3zVpj6WTlmQAA3esK8YZZwpSm71YY87GrULkZFSZOGUpdYgZehcebJwM11t/xuY4vclFdXg0wy4y41ug/0d1XE73osbiZ2CSWhciQCTmZLNZ2AL1sxVFF6Yz0WP379KQAfmlfbQ7CuP3z0y0mb5HL5GdzTQUJCmYV0FjWwc0YsqG3UDbJ4ux0Ej1kSpXirk2GUhmxpOWojKYk4vAEu3ZVsnH0BGGO3iIWN5VlwHa7EumJjUoWvQhQj6iYm7tUNY46YQhWjO3HXScBzMurAbcrWpT+lx9/i8UfGjmmfxmwuj/J+e6fljh9ogVdx2p/p3PVxOmgELXB2DDbddIyOdoT9cdCYaPqOnGqWPtgqSF3obJo+HVG/A03hvYOuUnwd4WambBZs662GeGTlg3E624v13MfWrznHurdgtJfSkB69BorjvWzmq6b4clrCgb7HVsTvmtLC4vgvKjQbHkhcgD6KmWJs0QYt5ygmcHdNAFfr3fy3H04bWRzgknDca9+jAbD5ETX2N0MGN/T+WZFbN0S6RZeellUc5jpVy2gJvofQAvGnNmZ2kyMbIAm09KDO5LmsXAW2lcnYbf6fQN7qCuD22soZpImW/t32Z6nz5ixpkpR2EOVyczBRT0o5Kmy/Skusf5cNggdLcS6MG3z+XcrbaJmp6GDlvdf5XqOZmTZXgn9nqlwsgGmN6Eeu4uSnoWgRL5j24CcNMC9dXuqbOICuwK0WdCTOnM58sXh62CBDw5X7+hn/euqgyUm9ky9oFTVOYdZHz8ViJefQZer3Ebx88rProrV3jF2s1COrKsVI4DFW4vuY0a8hJRfxmVcxmVcxmVcRhlfZKVa8lRjrHsKldjhqMA/0yyftIsCz46HFjmmCt2YLKHWCLQ7LE375DIUVyEf9utazZlIpgFzsxsmJiwWE6xOaFmv+NPjlRikr257fHO9w21HFR3BtgTnjcFIJJo1EaO3Yia/aCd0fLzWBnTGC/z74bhGw+YPRic4JgQFb2DetgLxJItZ5BT5eRb26NyB59N+heAN4ntW3LcJN68Il3M24u2/vKLrbTPCPx6w5nP+qhvFWWj7pxvYQQtpIcwcZTQzLAHWM26CXOMcKmElGyb0FDRJQSqlrLWYuzdbYoeWCiJsMhIHHbjrEfc3B+x7InLcLHu8WNB3SVC4bgepAt/2V/jl6gkAcNdUMtUP+xs87VaIe64Q2+rrqlTG/eoEz6ztQ79B82c2pWiIKFWud9EvAkwmEcgRiOskoQvhYYHFGyLE/fr2CSkr/POHN/I7Bb1YRnF5ej51CMEIW/xw7ADO6cx7i9jWbFs1071mUwl8bk/aRc/yVvKGrizk7GpVog4VfgcAfxvR3HH1bqOgNvbJ1oB4NlVpP3GrogItAPv5eoYch/sK8WqPymDlgqpUlsg4q7DMzLMWGsKiVkEh97lqX9tcI+gmJVrdzP+mhJHbfqZhbXAWQgENgUK7X+3xgpnuz8cO06ERlCy7hMzMd1+01vy8+02S6rmZoS2mN5T2Vp7pdRaDCjPR+Y135frUnGQ9EPtX7lVScnxVH1v6zjNHKNJQ8+9aUpVcWmBzwlRCBvh5IXcpg+4Tfa55zghLhagv8O9nOXJWyFmh33bywuUbT1Tz2cNc3Ira1VRlG0EBi1gncj+DezwtqAWqjF+PAL8Qp9SKybxWlG05Ncyc1BlLXlxul/RGFlZp08Tac9MJzkT8YUv9kKfdSrJW9clgz8zdqzd7bLoR1yxfmbN9AWCITvpsm3bEyBDy4dRWuv62PYNP84tJNg8pKqi7CS2zfMe3S0RmAoavJ5gmIjFr8ObFAV1Db/u7H2/xH/+R2MR9oHPoLP3sT5/uJJhZgQwaRKowKZn8kOvinrrEzNcZlsb3hRyyZhP57N52bw2W7/getUD/CvDMAjWve2n+KU094G+uCXN0JmI30QI7BEvMTF6kjM5yjV+0R1ieSV4t9/i4WyOz/Gi9HrBh+P7Ddo3T0Ih8ZfpxhcyuW3kdoPbFYEBh+CrWxUFDgqpzk4EmAQzd5lT7cb/78BJaJ2FfG53qRJjJxKMM1wT0I7tpDVb6amZUcM9K8k7jXUYulotBQWlmn34wYowC8GRdZCHLAK0zVPEpSEoY81nTPZuYtT0dG4An8jxngirQJF/6eB5n7NnT6+r+BdSfzeUk85BxAEiLdGaikYcaup41JK0otfQ5x5IVt68bO11yWPm4ydS/+039XZl7jeX6jDcJie0G5+2BlBTsIiCUfqbX0stUkawxyy9QY+1rQlW4GvlnhgxTdaZKTcZ4P5O2zDKTk8nUqplZRiaWr6lRo3li56WZPSFAC6pImHgeKNBzcrNe78zOVI9kCiEyq5XCeJuRhs+f/ftFLqq7D2voRQfoDLUupRj1D0p1qpok1emyG3HLbkIfnzaI20Z6QmiTpInk6AAohDv6u7FJ7AcXja9+qlnh+Wkpk2ZoMp5ZEpEBfH31LIvNA1bY/VdaRPWjwp/0rZAdlgkY7/mYCgj8dux/uMJzF7G8pQX6N/eP+P75FgDw6eMGpo1Y8SK+3y0A9j/OLsGXhciSx6uR71b7w+ZqgrGxfp91JM9fADgZhFR3m9t3GyxY8rK66/Fvv/saALB+fcC6G/HTA5G14ruFuBxh0DKBAUzM4T3NfKJVWcG8a+Ql99epSgcUgER6QKAQWbiPNwH9S/rzdEves4uXdH+n0dWUjjuPzga03N9+d9iIs1OaDHQT5f42NmA30iL1NC7lHEPSaFuPNetbX66OePtM5Zx/bgEFjOUrLSO625o8Epm05HctTay8WTNrj1gkNL2BXU7iCKVUFo2zsxHrbjxzcNpnOkelMyJXt9oleG+kmrY7K30wHWjSLNUMEsT5SnmNxdvSpyY7wOLSpBIA7vMalxC2jXji2rEml+DViJtNj90TXdcz8qCFOJJlrcitZzahT8zzCysKOi+LQbYVoaBEFch3Jv1sQVy0bFQU60iLTCebLKiV6jXMSdekqp+Fm5eRNaBm1fu8usus0UxsDJY2Ee013etxcJIalaNCmgeMA9UqkM9TqsdAGx6AeQczzW1q62IZm4ypRCm6DO0iUkmxSaBNGQBkBTUa2bBBV+2+e9Y1mL3IC2ebCdmoNPlcR60gm4BsIT7dZqTFvhyzf5MxvQlI/edv/nvpqV7GZVzGZVzGZfyFxhdZqUJnQGeYRa22ACBk2rUDlEtZYMtNO4lb0fXmhGkx4fCe8bxcocm8CgjeobmmKvB63UvfavAWA8Nr/bs1zFAF2dor2Duq5q66EWO0+JGZr/jjEu22iMvpP6lYoEt15ksrvdwG8FcKJ0VVyT/vvwaKSD8oJO3w7NiTdJj1IYOusgdmAUY2l1CL2ldumoBxdBjYg1c/W3HSictExhjFzPumbuX77zZQd9ybHi0OH1YCnZvXA6VuAIjPC9r5FthuloKRcw1wNicNd1Ri0v9zuJASOWaSD0YXxrsq91BeASYjeJZFRSUVzzg4fNivMTFDdNq2Ur3YZcAvXz3Kr0tZYYz0uc56Mbn4vz++xjRZBEPH//fnFeKhuCYl2JnEpll4gfpzVlIFKwVMTx00M2HVo5WQgvE2wW87RO6FL9cjXm2IiXzfHbEbF2Lu8WG/Fng/DBaawwbS0UKNGi0no5i+mqoDBCMWprd6aOWZs0clcN4cbgSAeFN73fGhRffJ1GSULiNdsYGBi9jtluLRrCYlvcxmWyvJTCCQJN1M97FWSl7B9FqqX9PXPl5Yzpip5mdOTrO+bnLUAirITHpsJbRAj0C7rdmxsavVqJ73YRVV1xLEkcjvFqiM2JK1vHl5wMBM7DSZSkv2GrrXZ7wB4TVwDmqBVu2sIowzkwWVqGouqoTVZqgGI5NFeG6IdQ1mZZeLYBOZ35RQh62pwe8z96N5gg/APr7iqKTQ7OvcFrvKEjZDZQJT8HvG8IKf45cRem+AeSvnMx1f5KLaXo0wSwXvDSzLChoXgCUkdSR4I/2yh+NS7PX+5vYBViX8zr2kn33aCHSjdEZaJIS3BP89hmUN4Q1Vs2YULQyl/+BfT3jBUo2F89iPLfyWyBp6kdEXOCnRg+n2FcYsD+nPY6uUV7AP7Br0NMeS6MUvhBLkCqlFi7O0kDz/+/UkDj+LxmPoG+iHGoQc37B2NgN659D9giZ2axKe33K0yCZAMcQYogNKPxBAHCwSQ03ak26xXB8gIzExLBvIhKC9wnibqDcGEGms9A/ZIUvs5HI1/U/LSsZQ0QBBUR8RAEYD3fPxtx1ObVuPbxPMlmHXTw5/fPgKzWva6WyWY7WR0wl/2hETJCUFY5JItyjhpEqMprE64TRNwB331B+OS+yfWO6l6H6KlMhDJuHrf9UYXmoMv6bnWKuMb1ZbAMCHfoN3+41IsLTKiA98IV2G4uCA5qSgZ65G2UDOKfFiZvf8jHuIS5jK9fkLV2yIzoulbiNF0oGkMGXjA/AixDdq2rbQpyrNkWsNgi3LAptcht8kcfhB0ELO0p42tmVzmNr6HJQFhj7I/fmi0ukSqm6J5SM/Uc/c9ZVw2Oz4OhTEdHZesak9xuRmCyyAsJrZ8OmMeBfQsZ47Rg1fXuypal3NgcLcJe7NVDi/9EpN6ZJ844VD0Lx3aB/rpjFeByE49qeW9K4A8qiJeFXUQTbLBhgATF9DEui8y//P7gUTAGUvESoxqdnRNR/vKrehQOV6IvIWANhThl8rjDf8zpw0bRhmG53PdVzg38u4jMu4jMu4jL/Q+CIrVWJIUgVRBO8xaRidJC4NgEAmXTOKacLz1OHkG+z2zJw8WDF4t22AjwqaQ37DdawSnaMRo20oIDcJic3IVVD48IFYFx9UBraNwGoq1wpUB5y52+QZ46+4ypTPZVsDAbSvO0QdMg7fVrE9kKGLwDsrmGJmzg4yhn1qX989Y93QLns70HcvcJv2SvyO0WZc//ZJDA0+vb8S4gmSAVbF0xRAb6AOFe4pO2FogpMEzk2V9GGCQmApQtxE6GUAZpImCSxXTKAohc2sUjIHjWwq5T9tEjTDovpjU7Mgl5kCBfiZMFsrFZW/DVjc97BsqtFPTqLs5qzgm1WPj9u1/O7masTdFclynvZLBG/gmEV9t+zxeKJrOw617MmZq9WSnWmAyNdmulYYX0Xc3NEx//HVT/KsxqzxYn2srluPq5qreazEEzroLG6sg0C1zbZmyAJcuczYtMU5Z3gToBZBsm7jcyMGKanNMKfqupNm8XC6N9QKmbUdCsyaris7V1jfBR5sKtpACIuGPlUCj56do7Bl+fekQrhbB4HizZ49iAW6re5Q/orj0wr56aBgC59sdr6FEFVg8LBJ4g7lbgYsOy8e36fdAqogM4OqDlGZn9vC+E0VmZquKYIufsXSOZeAH/hdtMDwmp+/u4nZ1tyqsBHTA6MemdodZ/F0hZE7qbNgcvpAuR6QlknWjAoUgChUk4vkyHWr3GszAN3jjCnNravpRuH4iyiksebJwD0rxIuj0uc5YjDIwcC6IItoSgpTtNJb0yaLqX3ngugRf9jeEPTJE6i+nYRFqX6/xOZJoX/Ji83aw/Ck602DJIsG4NaTHFNp6msBZP81h6ySgcCK2qszTdhcVgDUCU57YPlWy4MOBRjOkJw2tFhZNpo3Q3VlAWgxBQBkQJ80oqK347hqpD9sNaXoOLZIm14G3H1dZCcJC+fx6cDymMGIG1V3NcLz9Y0nSxq2UDcaaaYfVAYC+dpDTTiJLao1YdDIu6Z6v/dVy5lcJjmQqd9NzPW9kmsV7gPaTV01plWS7Fz7yyMQDMJY3bMUB8bfrEij6nnx1Lq6VgGQP/eTg1LV9P9+c8SOda8patzeHnBife53P91L33q9GgAOcN/tllBjfVWzrfe9/zrCvejlefzd9iXWbOx/mBo8nzqcGHo2BwO3ryboZTEo5uZzZ50zu7jZRDt/3qarjHDL98JmZK+h2dKxnd2zrDnrtLBAF1H6dqlNSMtcYdh5cpRCXVQLA7ZstHyd+VVU0LOkF6jz8y3PWNZAvK5N0Py+hS0OP4XBWh5BmwFfFwNkoHkux8yyKfMrtm4EQaR+k5GZN2DbIL/L2oT9x3Xtnep8tsku0jXb0ztRjNTmAeY60DW0b9k2c5Xw9T+9B0Bto97zuzo2cDbKJu/9717UvFrNetnyXuhcr4dhmHcWTlDuYVhWNrEKdFvK5tOd6oajMLILK7l5pmD48j1P7HQ1/X1PyrC+bKoNFQOfP/n3y1xUy5gGJz3VGDXFV/Hfyy4PAHanqufTOmG1GnDkuKQ4GGgONl/9AEw3QCyi/dkD265H6XHEqGFtxOkjLTxmb6TPll2G3dZYpdTURAwAZ1q85OrueHiZZHFpthrjbe2BuD1koSjSkrKTzKZOOmkTJRBZDZqOx+cfs5KYtg/Pa4TBQt3WXqbjit/ohJ8er5G4yrd3Ay0QPCK7ZuQ2IrsZMpBVlSnpjBw02g/1y5YqKi2yRKCV361mu/zAm4K8jLDLIJ7HKSmMbSefw4zUNA0OuVgYJiC9oklxsxwxBgPvuJq5TrJ4DZNDDBqLBZO3bJSUml3fYbelhaxdelytBlx19RosmAAXo8b+sJD+uVoFISpNwcoGJCeF3GW0H3ghclnMR/TdhNvNSQw8pmjw05Ya5qftAnpvZeGYSxh0hEzwYU2a4FLFplhJXaaeNv/uugAUHSpAm6f2o0H7UD87cd8+duTrHMvnvRaZSG4SVBtFpgObhUCFDCQ/s5YMCjlWUlqpTHVgAkwhxzSQhXke7p5NpmtcNrAz/TM96/UY2WBm9sDSGUGIKmlpnnDkuacsxXXS6Pj5GAcHvQiiqXafnOjZ0yqKfzidc/XVzpY0rXQNaGNYrD3/0z98j49HQkF+/O5eNpDtZoSPBsdiwYraoyVtbhapGWINM6devTqrVMufzaTOE4pm/dGsaXNRrqPpgXZbr3+5VsdfALHMcx9a5CbLRtfOCJef+7j0VC/jMi7jMi7jMv5C48usVLlR2S6C9MSmycKPVvqoKmccj1TZGBul8gLIFF12u6MWSG28BfqvYrXDiwqeZSfoIiz3cprWw5qa06m2BqavZt1FKA7QDlMqsYKQcYtuXrGoVCuN6Sqj2Smsv+djOCAsS1+GKPBz+0F/x5VBO4PGXEZsEyyHfisAv39PAeaLxYSb+4NUY4vVhOuWSpo/P9xi2jey419cD2L6r3XCzYa2t6O3CFFLUkqYDLKu11g/1R50WOUzy7i5RAKTlr5VuAlS8SuVEQ5OnIzSVO3dlEvoOOln7B1Sb6VSUEmh+ZpwxBD1XDmAoW8E6nddwPWml97p/tTi0wcqzdTRILOpyKKdKNyADRg+HVbo2T3I2ETtBkYHtE3ynGWTxFIw9xbN06y3mRWmr+j8v7nfIQN4PNK9GAYnxhDmiewApxdsJ7mdwxwQFmzW9CyV6nQetE3/AWfIibA+2TAFIDmXPdZn019Vh6PkslRI5fwxS3zJo5Ege6jaC9QqIz81fHxiyBYTEIy1ooqLBHPSiMXS0OUKb9osgQ6ICvpjI9URhXnX6iu5Cm/bo5IKlBJ01FkOa3nvYpsxveKXqUnUkmAXtai4Nw5AmwQ/WNhPTq5JOUf7VBngWdFjWnJe8XKUKt7+0FKV+oYehH/5L7+QuQd3lLwFAH60GD85ucZ5ZimKpKAmXZN0ZnaDAM095a/zfFwKdaA/u4NC+1Svv19Dzt8eAHfMCCUZ6D5LVqwZKgJit+e/l0Izqu/E5zy+yEW1aQNM6/Fyc5Tw7ge/ApKSBzidjLgtJe+qi4xNpKkr8//LEeOCtZwMJ5U0FDxbqPKiJ4WWdYfrbsRpcvLyqRkRJ6yp1yIvcKjkIXtScMdKAvDrGbEiVuKACgByTeowfYVnks0I14n6WiBYesGT2Ng7tOziY0yCVhnP70gOc/phAXzLfcHVCQ/HJSzDdOvFiENJujk20AcrAc/DsS6wtgkiD0gzOz36WUR5pfynBdxeCfzm9kr6q36VyTYOoEVh5m6jvCbnJ/6zCRDZiM2A+g2RedqWJEF0b414Npd70b9jP+XbEU0TJFIvnizcmhazV7d79N5i/z1jnBpCWNN3E9quSIzouMXXOASN6+vqDRyTlv7rcWikRTCNFpFh7ubREFmFr2NYZ2zu6bucJocpWPRso5lOVvSsWQPTXawxZX5GIJnqAqICoFG9Ywn65A1Zo5BNtZojWUeViQjxaVIIi7pInSWXaFDPtKSf2IxcZk9e3LNA/0Bktx/1aNE9VxjUX6VKjpvpWfNM/0lfLsOUdzcoaE5aMhMtlko2p1kmdh0UrK/PUljmKocr12+uyWU4dXod6mbtZKF7VVsmGwXflg2xonmB/0rSuEKcqES6ZquQGgpXB4A0dPJv1v/0QO2CHX2fZq9n7k1KtL7w1LqR/mSXBFLPuwbKK9IAg+LyZGOeGcIuRLLZoqcSzvTJsYHIpLRXYvvZ7DP6ez3b+Nfea9ZAy9Lu2P5sQ9PizA/6cx4X+PcyLuMyLuMyLuMvNL7IStWylObjfoVxoB1+7A2RXorf57o6wqSooIph+WCgBoPb39KWa9V4fH8kWNScDLLLZ6QAwwYGsalh1KfJYZosFEO+8yoEAOdDVhj9fZHiAAAgAElEQVRWWHcJOH2VRVKSdZYqJLssMKIaDKdA8A57tuP09+Tnm7hq8JPFzTVVPS82R/ncw2GJw/u1UPvtbw7YsH/tTw/XCN5A87UyOsnpuy7Az0zAc2/lvMwiC9w77Ftg0FAMS+kmikC9fU/uO+KiZCtJQs1gSzCiKKxhBcnfzJokRaXKGF9GGIZWD88LyaQk1qcSCDkDQoBJaNHrRkgjmBGf3j1cQ33fQRcY86sRDSeNrLpJ8nCfDksc+0bckd7c7PFiQWWIVhkpK+zZpB8Adof658LYLDmqZfcfrqJcx6Q1BduPxZFIV2KOy1CTrmYhw0wmsoEEY+uokFANE5Kr7NBscCYbyTaLQYMataAo5WdlqKQQXtSwesycquaMbIIQZjjjWGVR2eaZSQlgTgpNSeNZ1FD71CQ6ZiEdDRpq18mvKudlDxrakykDfRC1Up0YBi+ZrEOtaLMm5q1IgtpKYrJP9lyapGbOQ4sgIRR61MSIzbXqLdIbM1UP38QVoISZW3r3AEArwPdOzEnODDVWM9psb4jdXdjzy0lCFSaT0T0YdA8FbZj5KWsgWVWTgTSdWznfUnHqQM9Psy8IGs1vAHB8Q5V2uTduP4N5Vb2+KLKhcv39OWL3OY8vclHdf1hBLzrYjUcskIkCmqWXnlY4OmQRiAK5yGGCwuLbvTjfvN1diR7PDgpBA7n0cJoEFcospmQi9N7AnxpYnjxSU+Gk5GgxFh3ZMgjtPGw0MGOIwuszBxf9XDBjZj3yQqHiTPPpFeKkYXihe3n3LLrG4+Tw+D3bI7oMc10nxZQUPv7AoeG9IWNujus6jo1Y+fnekc60m70d/KuHXQvzxKbtmSbu0qqOUcE8Fa0Qv8AzJuZZunT+//j/qBA4ZN7uDWILTC/5O9iMyHA79rbKLGymtJup/r0wI+2e4ucKa9UtPPojM3UfG+RvByxXNPvlrIQZ3NqAWCZPlZGTxjjS9/5pcPhgaXVctBNOQysyrjy7mWnbwBXNcEeTTpmE1TLI9U5ek5SlWB8m1MUl0jX0VzP3KL7XYVl79e4A5KWq1o2xbkZim6knVyDOpORaNVtd4dJVPmONhvvZgloWTfHUq5OpmjT0UNnoOswgZJvhWPNdFr1i2J8WNcjCmIx4sDDMwjdjvQbIgHusMqKwqBsGOoF6iuXnAJD0PHyc4M6yWJqpBsaTZSEv2kcFf52RSv+yt7D7anWYHESnmQ1m0HntFccun4WPL//jFsuWVu13P9wBCci3fG33tgaKH2tftvSKS3B99EaercWPFioC0xUviD3kXjT7jNjWZ1DFGmtnT5UJ7Ff09xJxZ4YsUpns2NaxaFNn19SvqzOa9lQ4SNC8IWlSnM1nn+u4wL+XcRmXcRmXcRl/ofFFVqpoEtAkyS4EiOE7fVyKThOAMBvVMmDBJvlvrvfYuBH//BMFP8f3C2TeVXZ/+4j/4es/4sji0f/5d3+L1LFXbFTwHNytgoKa7f7j3x8lumvRThQNx1mXaTIwzMBNnrS0xfQ/60zwKsgYXyoFQ7CwkC1drtmibYS2bHoP4P37GzHbzyZXONYlKJUFJp4euqqd492kMJsB0aLmrOBHK37ISKA4KTARQdeKGRrCgM6DEa9VgNijZahQoay4SrXSUBlYROTiAXuwghrELhMLW1Tp1eTbTJXdmq2COlZ9K2yGYujNnBSUAxSzl0NfWb2rXz1L8DtA4dKeTR1OTSv6VT/ZswoUAOX4Ahj6FfIiQjMrPCcl91MFxaxTgtJUrDm9eTSIfH11E5FTdcjRQ4UYsyGotsCpYV2f7WQBWzyfFxy8XQLHhxnc6zJX75VIVKLHUpP/H/bea8eyJMsSWyaOuNJlqIxUVdUlmlXTYjgkiCHIIcGH/gh+AT+Dv0DwkcCAD/MTBAg0yAZn2N3D6UaL6erKEpmRIT1cXXWUCT7sbdvsJueJqIeJCjcgke7h9x5hx46Z7bXXXitbwnGTMTjozPBNKkwF/JvQgNh6+EqJxViwBcIyZKP6YJmZekb9+smTOxzYoOLu9Zog0RRN+8yUtodC8YjrTbXPUaaMiR5HfqfTOkiUFmqKJhOMTmIZ6X4zWWs8jXAnPiMdm8LvNCrAZ+P2qKPAtylKBqhes9oD2+8xerRv0XUpjFWADVC3qcg8s7LjzOc6b46VFD93tW1Qi9gLoQFC7vPxSDFLRQiBqr4viFuLTGgyI/WTnugau0c5rVDfA67N0SlmGSqHKuwXezrXyNq/vgExiguA40NtH+WiWs9HmLnG45Od/NvLd6fQp6OoF3mn0ZzSEz5f7fGz89cAgEY7/Os33xMx7PMf3OAPHtHfOl/hT1/8nhTtr9cddrwAuG2F+pRG77ivoWuPL55SpfxPT1/jZ4uXAIAQFf7s7of4a0++ox1qkVJM/0/XOHZZBEHySgCxK2slguOl3KDvLJUppJex9VBJpDxCILoYAD/Yo4mwXLTl8yCD6wPnpt1oEIcMM6q9lRfJzwq3E0MvZplzSi+mbzgvXWwKxOfWK4GW5+sezmkMe96AzDxCyjOxebzAYxGyYUqG0UBmwAor895KUTsUQXSyGJ2O+PITembP5htcNjt0vIH6t+FT3DO8HCaNvUvlWAFNO+Kwod/1bYU6sSEtMC0dQtqAdLm0yi+CuPGo3kCNkLII1XjJZ8cI8gnlewtBizRjrCK5EA05ty4+pjrmiS+AIPdDypkVC7iJtFixeIg+ZKWu6STL8AGcA5Z+VbJxg1cs+8efdUrKzgDA3hXTkIIYmCfT73QvYe2wXNPJu7HC/dck32MPmkUHGJbe5OdK/ql8yxPdZ5nXa+75b4UYBkALc9rIKZYHTUIrUEDF9INxnctfQhNhNwa6XKQKVrndZJeW/WdR1MuqjRHma70hb1F1ziYIxks+FIEqC2SD3Pos26iQdwgRRzhqfZthetksJYGqff634Yxg7mqb+25MY0QhSzNyn/QXSu47ndvN6PwlrKuSUYNWAhlrVoqyrEzV3PKzelBU+jDb09Mt7GLE5A3eb4i10M5H/PGzlyId+OvNOX54egUA+LS9w49mbwAAN26Js88O+PaSTL9fd2v87ftnAAAfFD49vcPzOb2pV8MSNRuWXzR7fL0j55JfjI/RtBM0Lzxb1+J//fq/oO/crjDdN0JwCk2EuqAXrG4maB3RsZrT956/x0lNo/TdYYWR7elu7xfwk85RoNPwG54RTET7yV7qcd1kqIYTAFTMZsmT5lxsEbmnSdErqCkTr7ptC8Wfi0HR55KsnMqSbmHuofecY3LqKO+m9jnSTjV8kgcuDKMX54fcb5sZotPZJL43UFw/rCJtNCQvbqNsMqTuN53b0cQD0KQwslJUtBGLr61MeFNQsJw3/XJ+jUOo8astGcgPk8ViRTNJ0ncFaNGbJnO0UKQINFSAvq+kFAo4LjMQ020dgRoSiUSVI7joFZHrRHYupy6Tzm1gGzfVmXyuiEIOUNG/x9z/MnFH+i+Ng3pjioWZpDgBQlRCULm/m6KExiu6h4KwkpxR7Nbwsy6eR+GGkhGWgGo+YsflTsrnDZMCW46xjGCwkE1SsBAd7eTQJFHVtvhcRRFWrn2N2aVmw+pQaTiORNQBKDedyDx2TxGyqC01uVaXNjTAdJoejpKINhig4ms/PI84+dk1vndKmzcXNL66ISLkWDWIMw/F+XN1X+Xysl4LSS+5yKTrN0O2jUz3L3q/DvBtQjby5wDqi5oX2FAoTJWaz3TfxzW8eixIXXWOdvVUPAvkMr90zGCKDc8H3B5yqg/toT20h/bQHtpvqX2UkerlbI9qNmFhR3z/5D0A4J+sXuIf90/xi3vySf3J2Tv85evPAAD/UD/BX80/BQC82a5gTcD5jLZ0d/0MLqkwqYjeVfj53WMAQFdELIdljccz2vZ93Z6jf7nEL9/QVu2rxWOY5JxhAqqTAY7LLGavDCZ2xNG/P2LRjPjx5TsAwNN2i7+8omscJiuwcIwKcZ/zi7AR6mkvxx/6LGYRJy1iFZI34xbrDOtQhBPl55JRHCcteU01sV8jU/zVyueIqM+iECEqgjVTYX1QWVSgjsDSYbZks/dFh3lF292r3QJbVnKKA0G8SCYIhY6pnwVy5EjRkcuRTZlT8jXgTgOQlKN0xIojzu23aygPLL+hz453DbpntAX/+80zBCjccR51Gi0OXJRvdrko3z7u4Ma8/Y4aCIVpQbRRIlA16pzTr7IGbthSGYXnyL5ejBhTPjtS/hvJABxAYLN0NSky95bjK9lGR5WL7fVIbFMxEZ+HvN2uqW5J37EP7iyXjKmTETHRt51CnHlUrFTlBouYxn9RugLknCzABtcxR07R5GhmWgaoWQ5t/Ju5QKEqZLi32rJrTMEWzz7G+byhot8Fvl7gKEJO+tIA0FxnMYzhBOif+mzsXUfJUbZXOW2RnH4kOvVFiUg4NjePJudi2yslpUPtT+7wfHWP1tDJ7/wc+1Rm5RXMppJx7Bchpy322UxCjwp6wJFLVdn/PldtITRKcqUiOFEIfXAWgzXDOZW1oBxq+nw0mb3smgh/BsnT6oLLoDwEKQkV9X3qHxVIL9iPxzn6D7F9lIvqRb1H3Uz4ze4c7w+0sP312+dYtYM4iLy6PpHPKxUxMS7x+ckd3uxXeLWht2B7vRCYrj3tMa8n7JlAYU2A578N3uK8poX4n3/2awzPrUDNm6nFLdupfbG6xarq8fUTgoq/+eQM8VeENe3eLrEzEftLOv6/23+GkOoto5IBG+cev//738o1f/3uHO6K7su3AdVqxMTyafZ9dTTxpDKCqEELkeSmigVWgWC59BJvjaituHlEeDSi4gWhqh3GoTBjLqBO3eWFLjQZ6lMzWlA/OyOz7RgV3u0I9jt8dSILCOa8YKdFdVBZWWfmgc6g2nC5iinq7KZMkDBf7vD8dCvP+9HZFre7xEIhmCtJq7lFxPWWxsvj+Rb7qcW+oxkpvJxBJVh3lnOcMTI8m6QrWw+dcuMmwncWhuG8MAvH+er3dOyqU5hOvNRNj5tGapxjG0g6cSgs0VK+sje0ySlyqlKOpXOOPDRM1Crzock1xtHmJy0iURUlL52FZfjXTSQjOG255KjTMKksZ+nJFJ2h/9IEOx0zLURmBKbzLPsnClkHg1AHgTjtXsmkfmSuDs7XpXuzGQ7XrCKVnFTcIsDuM4FntlWyiEDlumA/i4DKVTqzd1pgylAhu/sYACEvZspB+jTUtIES+NMp1LecVlgC1R/QWP/89A6fL26w5Qt5t1+iZmvAfldxqYyXY9ii5EgkHGMeswDJBqb3zrdUvpQ2sKZTqDmvbIaIaVFsRGPuu3qT68ZVIMH/BAEHm+9TeQU95prntGkCSHlJcrsNl+XwBsd2dHxV1vx+oO0B/n1oD+2hPbSH9tB+S+2jjFT/r1ffg5k3UCrij54Q6/ZPzv8OAPBV/wQAsPMNfr6ln13QEtG+2J/CO5OFBIISPdinpxucNQfsK9qt76daotYX70/x9VsitczmAy6XeyyqvC37EZOipmDQaId/evYCAPDZ4g5/v6TrePP+BPG2xu4tbaEXj/dYnhOk3FqHf/HkFwCA5/Ut3k1r/PJAUPY3+kyiTDVquKuWBMq5HZFE0g63ioQlluUgBVM26ihM1eo+q/j4RSD9Wt6eTqOFOxQnKMhOQI4yQmHX9ezJHS5mB1jGhl5sznDHGsRY55IFTJquJQnql6bwA0VpCdKrdlkpZloC1ZcU5jw52eK+a/H0nJgi+7GSEpjqngycU/QSLMSK7KZfYAoaM1ZRun80ihZyjEos7SKAMBgpmzHWw7E3athVqG9Mhh1D2d+Z8eyej1ivOxyYoOZGnT0vR4NoTRYc8ArK5WerBnXEcBa2roJovMaB4fyUPmjCESRtD0oQjGAKMpUC/C1Hpgz565Y1d42BT8doPKAAzXCw3RflQg1Fgel+plMvClzNYoTj74TOwG4yIlJtcxTlFoQ+JNjS1xl+jDpHQ8oR8iBcquuy5IUFCZJWywKFxjGpMVkuqfFN8TcDYfRaVqxK5ybVJPo51ASvi5Var3D4ksaOnjsYHi/PZhtsXYtf3NG7e7OZZ41sHRHn4chLNvuiKnmXzKhgD0Bzy9HoGDGxoUaCpJNVROndPJ5QlFpG/ql/fKvEF1VFHJPrCga4nhTMmFnOKhaM44JApkeO7Aui2HD6YFL+wbY/fvIt6mWNz2c3aPjpf9U/wW+6C3yzI1bv7WEmOcp+rNBt6e2gOlGP03OalGvr5XMX7R4HV2Mz0kQzOiMQr7UBPavgHPYN3gWFhuscHy322E70ne3YotEOZxVBxT9avMHvzSmH+tXZY7zYn+J+oGupdMCcF+afnLwVuPffbr/Af3/5r3HC/PW/qD6H50FuNvpocIcqCiSoChq+clznKjm4/LdoI1RUAnO5eZRJUnca0deYUi6s8JSFibKYqkkj1BGe4VrVepydUZ3Cqh5Qa4fes6NHmY/bmSzMw7BckhWkl724h5gnvGiQax4vJ5wsmDW9WeJ8eRAFpH6sML6gDZQFweGlkHi3Y2UkdYIQtEgT6opcSKhDskwjAKi5kxy2ez/LDiE+e5OmljZodeMw8uK7mA/ohgqOj696LZCj6TR87Y/uO020eqAcc1pIQxPFSAGDzs9bs4RhYt0WBgPfLUPxF5Mwu+11VvFxbP4dis1m6oPZckBtPe44dz8CuVzIK9itxnTC11UHYQaPu3lm3PYa9X02ItdThnurLf2cNj+68FaFKgTj2dw9MVxLA/BQEaM31Y5P53ljYToytYgpH6pyKgHIbO5pydKYvCipkMtwkldpGrvmx1u4HacOnJZUR6MdtuNC+BjTXSves0qRulVWgcrvoOkzbKtC5gwAgK8UQpXSLOANDnf3PeASzF1z36S5oc6lSQAwrpT0lQooGL+5XEp5+n7aqAedx3i9Ucc1wwNkcXZzhqWL832o7aNcVM+qA5pqwr+7+wx3nMvspgrOa5lcnTPiThKjkpIRawLqxslCWtp6/eL6kfyc2j5pkBa1hf5g0TmNsKC371e7mURzl083+GRxj1/uaafaFAnPl4cTbMcGZy0tCFYHPGpocd9OLQznwf6HR39K/8bb5MOmxZwNrv0MCDqTKfSUiQRRFy/Rd/gC5Y5W6kdTvqiNkoNTgaJHWTyrAMUqA2E00MlizZNGaHITWa8OmLgkaDs2uD4scHtPi5vfVBK9hLWD6lMeiS2sUoSlIXlH5ehF9yd8sQuHxQn1W2U8DiPNkJerPbqpQse/H27mAEsd+mhQ3etcIlFHzH9Dn/NNhemJg7dZuCGVOlQbjYlrF2MdgE2VZf6Q0QDtuF/TwrSKMDzOpsngdE2z//nsgFdxDX+15M/5TFZpA5WXpLxp8ZxUoMhUoiobZcFSIU/CelBU2siavnarj9yPgs3OJdVsAjiyduNcyjbsxsDPi3yuikCb72WajCz8ocrlNWarKQpO0U6xGFNnpevAkTZstEW/zYHhLBOo7KGwrlNAWbNaasv65rivtMtlHrrP7lDVXvFno/RdWjTcIvcvRXn5b97mBTcawH/a48tnVCqzH2t0PDcszjo8mxNSclHvMEWNbshENCl/mTSCzy+mPXzHYDzdsiPpQBGyWKjCMJ5qbm2S+S6mK4nu00bDAu6MPzaWGxCyCRTD+x4IJi+4vs0lUr6JaK7zScbT4ng6L76+PiaYfcjtd+AWHtpDe2gP7aE9tP842kcZqf7Zm+/DzBv0YyVelkpF9GMlakhaB4H2lMqsSaXob48WtNWbgkHvckQ7OkiJzeFuJhBSvRok8kUAbDuJALsfa9nF327m+D83PxD0ynUWFUe033t0g5Omx4bh3/PZAU8a2uFqFfFpTc45m9jgN+Ml/vLucwDA7OeNqKT4hnaFsmMvyh2OHCJSAJh2+QqFDJzKqkN8TJEwNFQiopNI/KjFDaV0UIEG4spJlHl3t0DkXJE/0ZhGC8/Ru1o4zFi4fhotpiRQUQUokw2Xo41SsqMOhqKG5NwD4GRGybXH8y0Cb4l7b3HXteiZDW3mDnjF6EIP9M8cFJcHRX8cjapBC+Srq4D2E0IN/BONeEvHUJ2ha0gKSIiISbyfo3W/dtInA6tkrdYdPl0RJHjTLzD8zSmSh7tyKitTOQXd6VyaUEDeSUw/fZYUs/JzSzlxKMAvA+wmRbHZASW0HnrUcv3GBvR8b8YVbPGKGKZJ1N7PA2C4JCho+EM2MbC7Qj1rFhHXk5SU+X0lIvQkApLvJVhAF+Ur2Vc4QjuIQUUJ6yqHo7wpFI4YvpI/ZJhS2LmTQpNE81nAIHm++nlEmOdjJii42qojAfnhPGQ0Zx7w4+fv8MWS3tF/8+pLKEZ3VrMeC8Oi+cMa//rVl+iu+SA6w72+jYCOIjuJWORvI5AcvustMWmTaH4ocsx6YulGDoSnWc4xJ1awiGqEHOX7eczKXIxySP+vCvGOSM8+qX+Zm0qunyLRHO2TkQjkGkMVEYtI/ENtH+Wiut3OoH1LpTI8+Yegy/kGMWhMU/5Z8QJobUBTeVwzccnonARY1COsMbi6YawlKKyf0EQ7b0ZsVJKu89A6inMJ9hbNe9asnVn4WR6U1WLEJ0yiCVC42S/RMJz6D6+e4O+/JTWn1bLD8xPixh8uG2x9iz3naUMNkVyLlmCh0sGmzJvIC4BU6sCTax1l4UwSYylfFOYhw3nJoD1BsrYgPxUThLoYsF70GHijMWwr0Yoduoqk+/jlrtsJqxnNCvdB4+ILgtC0inj97hTxnvoxFnZ9sVXwRslG4PLRBp+vbqmvqh5718gtXywO4vriJovwmM7VrgaM9y0i35NdTfDcH+NMY37W4YeXVOd808/x5pbIVDFoqa9MJSGS84yQ0qFoFbB2UgMKAGZF5/7s9E6e39dfPUZlMpSoggKKhUIViyXBpGkypQkycklGPZ+yI85dnYk+y4DqRsMWWrciMzmRzGHKa48vFmh2ebEJLOFod6SIlXKlaEljGqCcodpbyU0jMCEJgDmZEJwCXtIiUk0l3JldUlSkey1TEK6ot9Quk2OihUgFljlCl3Ro0zFCXhjckjYF1T0v/H1eHE3P4z3VbDoFzQPe7pTkaLUDhrNCkUvlc6mZw9V+gV++obSOf99g9Tm916Mz+PN3tAG++vaUNrSFdnZY5t2u6rRA7tUOR/nbsuazhFFjAaPbnueDhC5bwCfj94Fz6UU6SLgSY96cqBEY13khDVUUV6O0cVGs5Wy7gtgWs8qTb/nemnxu0yvgd4Co9AD/PrSH9tAe2kN7aL+l9lFGqqbyMJWH9wqRiUVaRdnFA4A2EYH9T2NQqAsB8Lv7hSi7zOYDzthb9Xo/xzQZcXZpT3shLu2HGm1Noe/kDXb3M0TWqa1vtUSEiSySCCWTb/D17pIvnIkmiRjhFDTr3j4/uUfNW8k/vf4RnrRb7KfsIjMwQSDpooqSETJURpBOJrXElUPFJRIImiIKELyt722OcAPkmvyKlGfSDlTMiPlkCZ411mMYrfRje97jdJmFR69uV2hPerm3s5b+9uPnb3E7Efb2F1efw7xp4NvcZ1IyoiPM6YTLU0IKLud7jExh/dubZ3Kebd9gmiw8P+v5ohft3sPrJcxBiym3u6+l2N6vAtbzHi82J/ycDB6fZvPxpLS0C3NgZ8WDNMwDkFyGGkBtrET59ad7PFrTMW77Gd5c0bHNgchfKWrz88yi1o7ZqMnU3sTjcg8PEYYIjUdg9nK1y0Ici68NfJ2h3PE0C0HYnYavIwKbLtj3lTBOqQxFy7moZCSNTS3vgbmzaK8yRD1cZGJSuKth77VER3o6Rk5KYQX6fhFNp7QFE8wTzK2nzBKGzhFt1OBoN7Gh83n0SDBugn9dm4+ftG7LEpsUtWkP6avhPHmGFtD8YxrDtvK4eX0Cw2IkJihsrpkVNenM8E1uQ4dshpHmAt1TNJfE/EvykPaAKmjy0yp71FKf8L8vi74AR5kFyQ06R64hI7dHqknJgCGbmUcgmZlHoHlvcqmczf1tD5kUKUhCGtM1k89+B8K8j3JRHbYNdIL/0qgZdX47Afg6wPCConUUR5g4UX6pnrNknHWixuM3NR2D83jDocpwpNeIfV60zWoC2ATcrSfYit7mmfXouxq4S7pfuSQCDnR8fsnQeFyc0STsgsbPXz0HAPz0+Wu87tZ4+WtajBsTM9zD55fJqs71kHoxoWKocNZMGJ0ReNyPmpifIHjTFX2la5+F+I2CjxnyNYe80IU2AoXwvrYRFZcVfX56hyftNj+ks7e4ZYuMH66u8IQVx39+eIJfbem+3v3DI5ij2s58g+ePN6hMwKKmt7tzFb65o53FNFn0LCmoW4/lspfr2N3NRXIRNiDMcZRrTM07heu7pTw3NxnsNnS9xoZs98Y52HhJ12FtkNIYTApx6XHxmO5tcAbvWbFpORtyLu3cwfe6mOEgSlGmV1AF8zMoynEBoHrTAFnE3bZCfc2w3C7nxZWnkpmsOGXETWRaAbOf3AlPYLjPtNv2PZVCAEB/SbJ7imfF+GSQBXz5tcbsOuDm9+mzCfoFSNKRmLc8KRelMqGJwrw2PV2vPIMxywqqoGiRC3lx8MVCKvZoKRVZuNakDZ9hmcNQZC/SJsAtOPfLp1bFmFMuL9r9I484y0pjs2cH4Wxs9wVWzee2V8wkXwQxFFeOmcsJfveAZtUneyCIW5x7ZnmxjApImahpSaUr06JYzCQfT9efoG09qFyLOgOCyRaAiZ0u15XOpXN6iO5FyeKoXGYAA1yONGRmcOpTMzBMXQjohzoilvVzH2j7KBdVjBowTL5IL6JTQFRSx6fr7MVZ1k5VyxGzNheBHfoG6yXNQLe9hTqYvEg1Hj7Jx8Wc31Otx2rZ4fGSF8SoMTDZ6WY3R9jbnN8tokoAVCvHi3bVOtRMBrk+LPDsgt1xugX6yZJTCIDhkkTWVqUAACAASURBVMpP6AC8+02/VkGi3egVHEds910FUwX4VGjudJagqwJs47FeUfTovJGI3DmNuFCY3szl+mVHO3ck2Qey1lM64vNTIuP80em3OBeeP9DoCa2ifj6EBq9ZW+4f7x/j2yvi+euJLMpKD9yKNzvrdkCMCoeJZojRGdqsAJg2OYJXOmK/bxFSxNUbZPcW0lQVabU6+67GNiAEjZ6l97CrxMklroKQ0P7gD34FqwP+6gVpR0+9Rb3Ioh/LeY892+aNo4XlSbgyHqcn1B8haOz2LVyXPLr0kRiD6SFOKaHUU+00wtplIYdJi6Wb6XNEOJ5yJJJENGYR6icUSvzok7ewOuDdnkKc0a8Lbd48qc/fUBSy+ynf22gkql++8hjWWjxylVOYvWCC10SLlpCrykgllvrEdG+ldm5qZkDeLfJzK/VrE0nPHiKGcwWXJBG7HJkmlxQlC1Mm7EwnAfF8hLqm51TfadR3/Lk10H1B96xbT2IHTLCzJgg5bn9oEAYtucdoCO0AgGiD+MmmnLgsPr3KjkQ6AkaJmEXZV7aL8E2O+vws9121L/LBgforbbKDpc8CtKBCHRO7RKxiyoQm11K9rBx/k/uRJBIh13KEKBTEJ5oXjvPAweba1w+5/Q4E2w/toT20h/bQHtp/HO13YF/w/6NpoqYjZOWYaAi2nZ/mWpEkD6Z1kNyf0YGk5/hvp8uDSBFi1JQ7S+UIYy541/dWIJLxTOE+LDBw7u58eRDJwnFmsPx0ECH+yRvx8ASA0dlsgr7oxQ82eI33twRvKq+gz0ecXlAkvD802SklKlGFAoCTdSfKTo8XOxHD+PbNGfykyXAcFOVIJBYJSrx9TxFjLDxAVeup/ChBjhUxPAHyg+1ueFtcByzne7zeEmP22WwtJUFPq3tsfIu/OZADz900x1VPkdLVdgl/wxGzZYUZEX/X4hP7frfAohkxcliy3c0k0lNNyOU7BwtMGprLS4quJljORPiad91NNlk3ywn+YKG3DO/aCMtON0074dlJLnX6qxefYmK4uTrtxeDe1B7DVEn/r+e9iFK8u1lLThKK3IUaZgaPVSVhgz9YhEYD26wqleC2qAF1m/1gx3MviIWeIIIAAOVRJW86czjjKFmriMFZvPv1Bf9OUC9wLIyvPdA9Dtk3dtA4/Yd8/OFcYfaGPj97F1HvqaP7U4Wo1ZHggAj2V0CVJAYjRdAC3Y45YhNYMgkytDkCWn2dv3N4woIjSfC+iG6DpQhO3G2mQppx7QCnUXMawAycFwaX8zTZTQhVgE4lWPnwZKKgMtzpVl7GrdrbzJ4N9PwyOpIZ8wh0z+IIM8tqX8NpFp1QntELzluGCqi2dDXVjqDfUqwhNdtRqYxbFigAP+PSxN2MCjp55AKY1tn72Bzob76YE0QgpDB6n1Z0r+nepnUgRO7DR38/0kWVm3KqqJuMUCZKLamxQYhFSkVsdzP5+WTV4YIVUKaQS2jUQKSWVEoQvRIHlXDqMJ7xuXqD2Bv0oIl2Y70sqpfzPeZ2hAtJY3YuptyDN9jtW5mgb79ZoLnhQW8BxdDe+HTCbD5gz1qxTTvhe4+pDKWxDi83a4wMN9/fLkRp6Ff9BQau14QC4mCOjL2TTBuCgr23Mqn5huocAYZPbSa6mNUEzRB1926eNzE6YrOd409++O8BAD+ev8EjSzjdPjT4ZrzEwFhQ5yt0jk6+v5sJLBctye4lSBnWi6NHbR0OYyWG7u6+zrrAvYE+of5WOsJvahkHasikMeUU/CJQ/hu0oaptzpfDZRJTe9GJ3q/WAfc94W2/fnOJcFtDsXLUdNfKbH72iMZQleo5o8qlPbvqCLadFi5PrgoAO91U6xH+7UwWCt9EWQzSwjI+4j65stJ3/UUxEfb8HiR4r7eIXOM4eIuXtycALxQ485IT9p1BdZfHB6LC5f9N4/b0qx71b0jPOnYdTv5NgzjwLH9xhukxbab2z2ZQkdSHAIKgJac3Zrgw/VvaMCSFJQCirpQWydkV0Nzxc1prMRTXqSy6ICCJRd+BNyFJQrmNcKtUpKlgb6y4uWgHkf1TISLwJq/9ZA/vtEiTeq+x5Q332eqAK2cQ2NJR9xqa4fGyiCSVrsgC6RRUkQ8lmD5/PtV5mu+4u5S5V+1y6Q3JNqojMpWQukALZK5bz5sd1RdQrSfIXghgNoqykxkVkRPTylLcHG2Y0i8MgTd5FVXHH/9g2wP8+9Ae2kN7aA/tof2W2scZqQYFJBHotFEyEcpmUXSnIJHqyazHuqXt3GGqUBuPLYvmh6hweUYR1juvgN4gJEKJV6J+klR0ACK5oAqi0LJ9uxQHkifnG+zGRqJTHxW6KW9NT1YdAhOjtvsWhxULSuw1/Bkzd9c9rA54/oSECbSKGDgU6PoZtAL233B0HYBdIkkssngCeEefYNFYxaPIiSCmzPhL22m39lBDpvSFSZPfJ8Ben/RjNR/xB89fYcZb7NfjCW4cQdmV8njRn+HPX38BALi/nwtzWh9MjibmEdV8EqatsR6GIzj3nbIl2AiTlHraKPdpTIBaj3IM35sM6TbkPZs0ePvJynOyrxtiQT5PrBGKRgASAXl1S8SqcFuT8hOzgBUg0f/Vy1NSyGH4MN7VqBhinHVKdvGhBvxpEG3n+XwQ0Qz3YoF6k8tVgv0OGaSKaF/S+NET0D9hsYZnB1QpBTDv8Oo3lzB33FfPBlxfE9z+fqQykJhEHa4r2MTs7jOhxh6As59HzN8wQ9wojF8QS7t+cY04b0UQ4Po/u8T2M7rPUBFMKQzUOY7cVjLllqJUIbLoDNWqQKIG1T6Tt7oLLklbFyxVZqYKo9UTAQegiI8INvz7LOtZV+8t2islsHf4oodNZhirA04aGgO9q7AfaxxY1MWagHMut7vvW+DlDDaVmlTIjG2NTI6DOjJZqDcKqoBxfRvhmNVbbfRRhCoRZ+SIsExlMHFoWhFKkf1PIb6oCa0QoXwdYZP6l8n/HiqG4gu9ZiGagZm/fG92V8x7XNKU+vfY65Yg7/g74Kf6cS6qRU41MWTrxYipq6RU5ny9x7yin0NUcEU9q1YRO4Z1fNAywcU9uXaIglCEDGztFZWUAJRLCSovPhGIb9kcPcGUDM0tzw5SB+ujwrIaRVFpP+txOydY+n4zxxePSTFIqQirA3a88HeTFYFurSP2dzNhEMemLB0KUq8JAKozMurj+UiWbgDiTX1Eh3eLmM3BQfV0wvgNSvrAzB1+9vw1AOAHyyu87k/wS67B/WS2gebZ7mZc4Oc3j7HjMgSlo+SpPPKiBBsRo8JsTrNJZbzA9NOholrOpBxVe9hTmvxiVBgZQg+NJzH8BPnakKUOG4+mnUQ1y+pAZVMAlKINBHixr5sJN2wA8O76QupI48xD1QGa4V/fGeiUU723PBEyQ/nphGnNZVa3NQLnohcnPZzTktff3M5h3/J1gPKhIgHYAXrHG5x5RLXJ4viHHw34n/+rfwUAONUH/Kvrfw4A+N/+jz+CCQr2C66z1REdw9fVFdUjV7zR0EOGGk1Pwu0AYHuyF7v+GfWrGSKC5U3Xz2awHSSP6po80VY7ugmXYNi9KibvCKXzZ32TlXtKF5ao6RhSMmPVEYRc1mFD5UVWFezWUFF/jc84L74eMLxnlaeNwvZnI9bnlGfu+krek9Y62bBuhxq7QytmG4t2xF1Hx7j56hxVl8udzNMO7n3eEMt1GOJ3lLXA6V5CTdB+Kn2q7483Fmkz4uZcIpUE7x1yDWvaPKaSulJ5iRngsinzBUwc1VFpEslh5ty0zAUzur60WUGAsL7LkhpysolSv02lPkrUwD7k9gD/PrSH9tAe2kN7aL+l9nFGqpG2q6qom6xrh0cnO4kCe2fxbksQ2DQZMaNeNBQpJki2GywG9kVMvoJlDR048lDvaxgu4o4H0l0tFYmEiHGgWjaphzzN8O+yGdBYh4uGdsyn9UHqW92yx+tbgnS1JsJVTCSPwcDcZghZtQExKRs1XmoqEZVA13pnEKsIz5/DwYoaTH1HsF+qb9Mj5N70SPWPfp4wtohHz4iQ888evxDVp51vROEIAN4NS9wOhENd7+cYJivCCiEo8SP1ulDtURSpptravq8wJdNsT88iMY+1DgjMDHbvW4C1laPTaFaDRIFuMuI8JsIObEm37xqB6dy5owicWcTDiyXmr/n4n3uoC8binEZ0CoFPYGZe9IMxGEBrUdFanHc4YVTiqlkicF3wYdPCNk7urdqYPD4qElivNwUUz80e6Dl1T+mz//K//pf4b/hc/+PVz/CnX/8efTAo+LWDT+PYBoHAU91kafSdVHl23/fyLCjNEYXhq0dgdk3nWn7doXpzB0x0ceo//xTDWRLNZ/s0qZssojTketRkS5gEH6LJ5JgkbjAtM+tZIrjI8CoIlgw1RZ4AMVrTEAx1xPh0wvqS3q1htKhv6Y+Hzx3mJ50IYARvcLKmz11tl0Luq5sJZ6sDVg09e6MCvmXBkXAywQ81QhJ5eDWDSf61TZSoL9SR5oZiChGykCe7tcT4VSF/Trks6OAWx1q9pstw7/5RxHQaMtt9VHCJZMiqVcKAHiFqSyX8S+IXxbl9IRySxl/ieDWQ56Qc5DpcRfBxsiw0vYLts9LSh9w+ykVVzR3UzGG+HIR5uWhGRJDPIUBiAQnGqWuHymZ4c3BW/Df7m8JE2CsopRCSGP5qFKEBYhrmJEcpZB+qzMRUnoroEwyyf7PAeEYjrbYOn87v8H4gmFGriMsZvdwRQMPX2FiH11cniKzKZA86w7MtvYkCdxbKSNO+kpxHWDtg0rDvOT+UlHv4ZFHnCTw2VBAPEDTYX0SBlx893uC//eQX1G/BYs4Y1dIM2LsGVx17hEaNqx3dV99XiEELm9ZYL4xK3XqAJSOVjvCDEahS7WxOTV0MMCbC80IUvuPAkxaNsHTwzsCnkiMVoVOpg6ZFe0hylRFoGEIeuwpxNAJf6VGhP+cJv/XADT/3OhLTMfmf1kHKrKBZMYgnmm5fEzQPUP6ar6NdDuh3jYyJkjHZvCN4tnSmyUxZFoNn6Pyv+s/xv7z9EgDwFy8+h39Bm5hQR9TvLMbLbAKQlrXxwkMPmsT/+dwp79u+MqLKFGqguY04+SUvKFOAvaU/6qs7+KcX6D6l59udaXD6HG4WydWk+v+uIuWiFw2VM5WLuCycXO6RPTxxVJohovnziOYm55+nZZSFyK8dQb6cygm/XGL6hMbqfN2jP9SYsWjHT56/EaeoEDRWZzQmTpoe67rHylIfnNd7ydW/CArdkAUx9FTAuk2AqnJJje50XsxsFuWIlqUUp3zfYj7go6hb+SZSv/G5rVWyafGzwOL1fIyYF71QR8ltArSgpj7VY4Y1fR3hllH8Zu2+yH1HGndJbEK7vBhPq3j0nO1Oo32XF2MVQA72H3j7KBfVR+dbmMWI2+0c48DlAVEhRiWqOxFAy8pJ8yY/6fuuhXMG/R2NGntvZUKbTj30asJslhM+zWNa9PabVqQHVeQ6uEWxyKb8RCrL4JdK9wYTl94c5gP++voTWWCezHdobT5Xqsm0hvKEsit8lK21ACIPpQ2DG4zUFgLINZ86onlnjnI7qaUcVtZrzT/7hs73n/7wNwCAn65f44Q17yrl4fmDr8dTNNphVdPb/eu7c6nHrSqPacy6zG6wsgnQOspi63vSzdU1R91PRjRcUuOcQX81g+X8Il0z53ZmESEtzFWgchv+vZlPaGrOMasIHxVaJqW0lcPdlkurDE3w+pTGhquzhi9GnZ8nR8xSujUVcoNecf48lWBpqR9u1hNJFQK4286gbiqJBnxbkMS41k8sutpis6OA8QSwNzR7/0//+5+gueFnXUWENW8eTkfUnw94tiCi1e1hhn7Ned49RVdTRd979OcaF39JBDh8+wbqjAhZcJ5KZkbmIfzgM1z/U6pjHs7OMc1zZCPjBzSRR5sjNV3WaJblFgHASJtWgNbdpOJDNayQ8a6nIldqC3s0T4pSspi1BarkFYabmYz/5U/u8YjVkEJU+PGjd3jUUs75N9sLbHt6J1ftIDn3ZUUnagw9gLfDCrdc9x0C1fyIzZqFbHTVlPPewVKuMdXBAhB+RbQBfqEB8Ht+gDjkuJkSwpFvaXFU+3xML0gYzQtJgSvqYpOmQJsTk4+TxpkZqKwGADCnWtSk9ORmMctM8rNIC6lv8z2bHiK5GCwpVqVr1hM9Y/87kJD8HbiFh/bQHtpDe2gP7T+O9lFGqu/enEDPWsBpmDVDPPWEzaGlyA0UiaRobt/XGFif1VYO02jFYNidlEksopIvWjrmD87e4+Doe7/GObp3LOLueHeY6OSDOsqpHG11FARK9VHh/jDD/h3Dv19GtJwDvt3Nxdh4ExTUzIuOMZyC55IXpSOVpxS0fRHvbj2woS1tc1VEr+A8VZlTCfn3Us9+OA94+uwWZzVFp7/YPcYPl+8AkFDG32/IIeamn+PT5R1a3tV/7/QGWy6Mf7VZSzQOANpGNNynR5FqfbwnnDUTPH9v2NeU50tMT58Lz3WvRBELB0N9kkQulsA4JW9Vg6p2uOOcmRuOXxdlInwq9dlnzWezz5FH1AC0yvanDAcDhE7ENogoRbsccLnKSkbfvD6nz93WsAcl0YC9zbBf1AS5J4EDNeUykWkZj56N3WXtX7cMaJ/RuZ6f3SNEhQ3D6IdDgx88JeGGP/zRS/zZ2+/jFZszLL8dgStimfvdHths8vGfPcX0Y1LBevvP5uie5Ny3GRQq+hp8C3Tn1AnhbILa2oLVWzDHixIMGPpZSnj2hVC7ZWYwQ+zjPGbvWQfYxGWwEeOFFy1tu9cCxU/nHr/3w9c4a7JTkuXwX6uIm2GOv2N3owjyuwUoFXTS0Fj/6uYSP7q4kjH4DzdPcJ/Y6O9mML0Whu60itBDUmhSmE7SA81qR6klNCQ4heZXlWgZK4+MeqgsYq8nNksvDAdkrvEKaizQjCrnQxMnRBAol1NUCRGh6wVCyN/rL6Ncf0oNiECFKoQnUEDZim5W9cc5298Bj/KPc1FVnYGCQZx5rBa5ztB7LSuEtj7XLnqNFdeGuqARvIZLMO3ckRwhiOCxXPY4nXEuCREvN0Qe2l0tUH0H4kkvd7CAeUbfsSZgvGtgDimZAanv6w4N/KShlzRr7voGGy787N7PhXwQmkDuKMkcYJ7rT/3BAjaIqwyZC/DLd12juc1waTJnTteRLW74/6lCaK+yesukMDqDv37/CQBg1Qx41pIUzd00x/uONgT7ocamabFmHTqrAwK/tdNkoHWArvKknKQZl/MBkcssRlBOK9WmWhPQd7RxiROZzqfJQ0VAd0V/i7ITTSyKTQWGQwXLEPJy0aMbKkw7hu1tFKg5jAbYWLQMp7pZlOsqTaJ9G8nuTRezBY+ruPRYnHb4lCfouR3xN9+S01B81aJKhs1pA5Psxgq1HLeMmEwm35i+EE/35HAkC6vNBLLYevzBs1f0zyrgl/cXuN3M5RJf87idgsHV7UrSEpsvajz+9Zz7WyOe0edu//gC2881pmUmEknpCigPt/09PvfCS8mRuaE8vihtqWLMqZzvE5Wf3ItSrzk9mlBdVZmYl410iAuQlKUukoA9pwE+62Wz9tnJFk9mW7zcE5x90e7RM4b59rDCrs8+cfNmFIOB1jpc9zSmLxYH3A0z/N27pwCAoc9jR3tSrWJfCCL3cE6yrFlNC2BK//gTj8Bpo9m3BmbMm1ntAMWr0HSanWKCoY25WC5+Z6FSrhif8ygbrXQsce45UP0vwHA7b9aqPsK3SmQuo43isjPx/aW0iyo235RvzWM6VFHGaqgi3DIgdCX54cNsD/DvQ3toD+2hPbSH9ltqH2Wk2j7bw8wdrMlEof1Qk+j8fwB+eHa2kbKWsWvIzDxFqnVAvSaCglJkI/aSCUMvcYKhS6wIJdAH7b41/CmFHtVyJMFtgEo7TIRjEkmsAyxHpu1sxG63gOHfJ2eysbqJwjpGVMByAjjaDazLCmTijJggVxGKVZ/aK30E94SqpPMXNH8+XIKhzJDp/I//6DUq40kvFsBPzt/iaUOR6tPmHntW1H6BU0zeoNdcwlPUECgFTPtaohmYKHD7UHkRoLcmoJsMav7bWApXVAGx0XIIPeaIJV0zAIxnAfZRh8BlM9ErITsd+hpTV4lxNFQU8QezNdATMCX7LjHRpkgxCe9HE4HGC1EsOC3a0JenO3x5coOvN0To+flXn8Dcs23YIQuWJ4sstpSl8hFmGquJdv/Cwi3syxJknyKz+r2G6dgf94cD3h4IM36/W2B3Oxef3vraYPcFs88PDfzBCoTqG8BfUojilhe4+TELjDwmAXo5t8nlMG4WSPt4ywbp7wrx2nTNyNcsz3BSWYc5sacTfDjPaRK1N6RClLR6Q4YhlVcS0cpxmQ19frqTcbeqBlx1SyEZ1dpjweoJ77sl+qESxawYlYy1/VBjPcvs3zfbFTomO/rOyrtFfVJCvplsNp35rEvNfqoJ6o87g/qen8uW9HaTQ2K9iegelfBplJ/L0hjqh/z/aAvPXRMlYlaeBO+lVCZkRAQa0PsEyyu4FgLT+0WA5nFlDoqM5oWQFY/KvASGnmhcTycFehBxDEV8oO2jXFSNCTAmYCrKZpwziEHBzFjqbzaKF+KqHrBh+OdwM6fJnidJ77R4jsaRvDj7xITTEZHpbGpS2Z/xwsEsHRYzZo46Dcd5PH+gFzHBKXY5SQ5x/2oF1EHqJ73X8AzjqsL/FQDioXi031EqUaPKNHp17Dby3YUnuUqUg11Pma0H0OK7/O/eAgB+fPoWe5ehstOqQ8UH3flWcqWVDlAqYs7s5UrnspkYAVUYuscYZKQqBfRczhSCQlV5dJzzTLJ7AKBMQJh52RX4Am6HBvxTztHaiKryGDnH2i4HgZrHXU1s3eTXqjJM7AHEXsvCF6soEK+aeVk4w0iqVLHo/+TderOZ493VGppz7fWAnFsccx1wtBH1nZaNS2iiPLNqyzlVrh0tyyzcIiC0UWqISfmKN29e4Zu/pRyhcgr1CKx+Rd+r9xHD14TL9RcKi9cRy1f8nDYj9J5ZyX+4RPeEx/SKVJ0y6zY7+qi5RxwMbHLSmYAgcoDs3pJgXmag0s/xyHkl2pzvq+61LJbRkqduuk/TZUUlvywMwOcO6A2ePKJNng8aM1ZN+9XtOZwzOFnQ7uSL5Q3+8f4xAODFiwuoKmBXcXlWUJhxRcC/+PSX8lx/sz/Hbt/CJzekQ7H5HtWxQH0A+kdpQSkKTgHxCQZoXKXaTTcHQ6j0t3GtZExMhbOMdseLrAoQeVCAFnMpqfF5Ix1VXttTk1r0KRvSh5rSDklO0d6bosyHNoNpHLjZsQpWOWdM63BcSlhFRPfhJ1U/ykW1++UJdNvCF/ZL9rqCqiM8R3t20WPNmp6dq6jwH4Aas4sJgCMnF5hIUVWaaAvRAnMxAOSehYqru9PkHYPKZuaOBlei0bu7GoaNsGNLublUziNhGJBlFwHYjaGXMu1c5w46kZ0OFubeZM3NohZtWsWjRTXanJO03zF09k2OYg9fTvgvL18CAGZ6xBgsnswojP3Z4iVqPsgQKokMTpsOj9qdSBPeTzOEmIrrNeVtZXJVqLlMaRyNyCUaEzAOlXzssKkK7WIFTEom4WhzjZxq8022s5EWcS6L8F5j3Cd/Kg3YkJ+nU1JvmvLKYnvHzx6gMRH5PnUVYKzHuOXc2s7Kd52tYHe5brJcUGJhyG166vuYIoi+IJ44YFoUm6EIcUPyywDYAMfjwK5H2VzZ37Ror/JkFw1gRvrc/O2I9S8YffER0ShMpzTmrn+6wHDBoijLmC2+FC8AqW7XZfuv2BugDlBfMAlIR7hbri2eFEekqQ9i3gAW2rVQqdyG/uYWIechnYLpMgko1CwhCdA7ILZqCv/kxy+wn+hZDM4KAtX3FYyJYuP4Zy++j27L1zgYRK/gZjQ+v7y8wR+e0XgPUeFv7og/8OurC8xmY7ZZhDmO+uooUqW+UXkhtfndre6pRjURrexW57wj1yOP7CDkmywBqHwh3ai5746s4PKGMlTZDs8c8l6cJAqza03U+R2fmnhETNQOksONKmsmQwfKiycHpE7LuaOJQp6aHrMoDvNFYhVlLHzo7SGn+tAe2kN7aA/tof2W2kcZqfrTCXFmgElLfse3EaElVxIAeH5yL1HVTTcXJnCcO2gb8u8hl8agDmiWg0RSlfVSnjGNFsrknKd3RqKe2BuYHTOIvaKi8JR76TRCm/MOR7DorIA790YYlMlcW3bCk5bSD7ulvLFLwhMamYkJdZSPAzJDsfw3RNp5J8jp937wBiumCT6pNrio9mh52/y8usFK09++Hi6hubMetTtsXYOeldSt9rjpiVXq3s2oBIbv25yOwvD1rsLAzjPNasB8PmB7n3BRJabqMJFgWM6RlXllqAzL7+9mQCRjboCg+NT3utOItcrwYe2zabtXQMi5vhhyP6q5Q8ORdQiKnGS4H2PhIhON4giG+7pTRznJapuhYBUgBurB5qgkVDjyAZ1WOMohqsEgJuGPFzMs32TWc4qEp1UWEQAANzO4+kPKie+fR2KBi7JOFmcoc3aUYjjOswnjWQGmdcIb8M5kZiqAeDYJq9p3FmrD8GlQ4KGTyzcesepTHeUY+kAQZoro/NJnKH7Q8p7YdhJ3KQD4ZHkvP2+7Bs4Z7DY0lmJnBHlYfrLF0/UWJ1wm9tP1axw4L/KL7WP8/FfPuCM1Jj+D4hIYdT4iMrQfqkh9ws/QdhpJG9MDqBgatzuC0OvblDBGjvhZyD6y+lIprKBcIejAwvjizvOdXKUec6462iwRGdWxEUIovHmBYqybiKnNzGy/8OIXrG1AGIy4MtlOHUlLhgv2Jq49idCc+XSbpAD34QeqH+eiqvcW2ltSF0nQRxtg1iNOVvTiTN7gpqNJvp+s6MYmCbu0cIaNngLfSQAAIABJREFUgWHSj2sChpuZ2L31ZQ7ossNJKsvxGsNkRW92bAwmw2UbHU84PJb92gtEUt1pquVKeryDPrJmS4tQmvB1gqFiQRpZ+uMXrNMyYaFQurEdTZ6hyAGVFlrjSYT/Ad3PquqxZcfic7vHz2YvsOBEUKsmXHkitlTKwzKGfNUvYbUXFRqtAt6w2Tt0JDWdpDQUFJU7gXLYLRPD6srRgsq1tdoVk0UNIChRKFKVkvwz7ipxJ4Im95yyZXUrAKNCsrXTiwm6yZ/1zgApp60jquJvwzcEkepRoery8wzIYyIqUndKkFdUGU6mBS8vqmVJjYoQmb8kGTmecJ7tkZfcYnVriBx34DrbtceGoWFzb8Xgfv46YvnGwTf0veufVth/xv0WFPROwzA5ZlpEDOdl/q5YHE2Ukh27MTkXuIjwTiEus/pXWuh160lu8p7H/6ilr1TI+X5f0+SfHaAyfBpthHsyQrE0pApK9rn6fMCTc2J4PZlv8Zu7czxa0M1cNnv8YvOI+m2oUDeTwPQAMGNJylk94aTucMLlX6/6U/ztDZXNvHl5duQ2hfWU8+fXtaQcqMwlQqV5xOV7a68zQTJY+ptm4lmqP07f0aM6UprKxB+VYVyT4V2AN178HdNzeimlGXRetBO0LnXCJv+br/O4RQRB1pdJPtWLa4/f1GjeGZksfJOlJf06cw1iUNA2QnOg4SdDm1L/4a+qD/DvQ3toD+2hPbSH9ltqH2WkanoFDdrxJTji9GwPrYNQ5X/59lLcW6Z9LW4kMBHVKlsp+CbAJeJSpJ108uPUsxy5uNHg+pail+gVYmdhWc1JqUhC8SAoOM68lDfYG5tJKBqAjhIJQ4GEBUDRaYKQYxVhhmLHHwG3yiy7apP/Vja7y2SkZFasM2Kd9WUbupa0I397WOE/Wb8BAJzZPYwKmBgvvHJr9JEipZ1v8L5nkkswWFU9Kj7B7TCHEcasguq0RE4hKkx8vfPlgIYZm7fXK1Tf1phO+XunE2yTxPYDpn2NiX1TYSJUMgOvInQiSJgIb7NXafSKTAcA+IoiTMXw4eXpDk8XRMDqXIUXt6fCHj9ddHj9hkpj1E0lEYq3EaE+3n2HdbIPibDvaiF1jIsstm+GHN1Oa/KoTM9ColTksqdUmgCdyTDhYsJs1QshLgw2i36oKG4ns5sAX2u8/xm7/Xw+SvRVv9dkbn7JfbIMGQasAkKKLMoohltKHZhBYTwDgk8mAwEqMeY7gxghakiIOfpVPqsEhYruSxVM0pAYrecD2nbCkOBmp9Gy+P1yNmBV0/v6j+8fI0bgrCWs+/+5+hRv35CLzJefXeH9biH9DxulHO58eYALBt/s6fl+e3OK/oa9UFdT1qsdNdBn4mKYZVavchp6r9Fcc1pnzJC9b3HkAEPervx861ySkoQThORfkIajzimByApTyREHCnJNKuBInCRUmVQUwJEpH790C1Iuv//Ka/TrSQRxhsnCf0sX3G6ztm+61qRxrupjYQdtPFzyou4MqoNC6D/8SPWjXFR9ExEbEtNendILVlmPXdegY7stBIXAEy+a4uWoApTK8K+fOZmooleILZlSA0BdwIFuMtnI4Z6KwRyXvejW58luNQFjzjlFWzAqe4VqUpI7cosAu0neVeUdsv1am15oheY9y+mNDDuKa0UUFihi/vckrJ3U/EqRciiSQEv39731DX61Jxm7KRrUyuHKEW71xN7j7w6kEvQX77/AltmVT5Y7nNUdrtlx56vrS0xf088qqCMnFkwKFctBKhVx+80ZXzwwPs593H7dYLjghbMJgFdSFxgWWYLRHrRMTFQfp4D0rKtcF7xadpi8we4NbQTevjrF9ISON0wV+n0tzOBX77Ma0ZFi0EFTnvy8xG6TmhXlrSR3vB6lHEPvKnjORRm2NvsPOX9QOU0uXzHLSTaDIehcApYeXZLDbKKU4fSnGpvvA+MTnpWdRvPWyLn6x0EmaH3QAiXCxpxH5txlqsukBZHPGYi565c8qZpIZVLpGE7JJlL3uoAxkaHOSOM/Xb97PGG2Zseg0WLoKpyc0Lt8sThIvWnnKlwfaFwdtg0uLnb4dksL6bv3a1RzWnzf3K1p85E2yDpKXtwFjV+8v8Q45FKup5/fAADe366guEYzaoJVxTwhQOBe5YjhXOZAYzH7JllL3/BCmrpqyPXV0qcq/y555REZd1TMBk6bb51z0nZHOU6flK9szPW+Y/4c3Q+OoOayXEofDLb/yO+hyiVeUZNhgeTWZyGXpAEIPOepxmMa6gxf7w3lescPf1F9gH8f2kN7aA/toT2031L7KCPVZFu2+nyDmj1I90kwP4k63DTC+kQhSB0j1VFOfYpOda6DiwqoAjTvdqepYPgWxf/RRCISMeSrdURI0OekoRovdY7qUKG+z5CXm0OUmZRX2RQ4ZngmWLZt6jLhJsNoBdkDpBkrTNKCVQoArmGCBZgIodP1E4T69JQIIF/dXeLxgmyx9q7BV/0TnLPsy991n+Lfb4jUEaLCjNWQNkOLKRj8+g1FuPpFKxCsXwSKegpS2HBPYc84aGixo1KoXhZsaA3Mv2UI8zEp0whk7TJ5oixOpw6KpEAFYLnu8Xi1kz+93awkMlheZIpstyemYtgzhbYKmJ1Q5NTva5jXjVzj+GUvXpzjUMGz2o8eNAvI8zX2Fs23mSiTYL9QxSNYLugCaagjpgsHu6Lr96NB0Hx8G4jZzGjG7K1Cf8mM1m8A09PP++cK06kXNafmVgmz21mgvtei5eobSAStr4xEUcN5QHRZ4CRUGbYkQQBkyL3QikVQUL0RqLgUgg/tMQTpm4hwSf3YzEdMDB0u5gOW7SAI9F03E1j+k+UGL67OuD8iautw39FY0iYKwSaESGpX/P7P5qPYQl5drzBfDmhZrCVGhbfvWOR2m9WhlFNQk84kIXsc5c3eaFEhK9XKoirhWB6bqXuK99itIjG7BVJWqO8y/J7QnTReUv9rp+R4viXhDdHg7fJ7nTxes41gFEQEyFGr6RXat6X4Rq4fTt/xMz5Ik+uJY2ExGXsDNeVxVd/RPOQfTMo/zLb4fAMzH8h94o6gIcMMtqR2Y85GEeiOXiMyjIhJi5NNaiXVXJlI5RsAEHVeTAu5MjX30LXHepmZxnsuNK9aehOmJDFYk3g1HZ8mnCnR0J2C2hdwGyPXkdl/qWRCT5BNQcqbhuIWZILO1QaYFhHuxKO+ZvmxIefyplXA6ed3YuhudMAd+0Za5RGg8LqnSefvb5/gfk9/O1l0Uqa06Vq8/PZc4EN/OcEkAwCQSIVsSMai7yYl5Qa25xxU6Z7Drb3SCAZSRI+YVWSgCsWd8wn1bELb0KJUW4/e0QEfzfa4MXOsn9JM6INGx/B13Tr09w3MllmPl15cglSfi/59G6Fuagy3LP4wKHnptKPNiubxZHp79L30PJJYfjbszpsktwhQrReVLW1Clq6MCrOvK5z8ijcnLuCEVZPuv2/QkWAQqd4MSvLzbswF++NpoI1c9nA/8tgdznO5lznoQuw9q3iFiko/5Pm4zKiGZ3WvxIQ/D9lbtc8sXnfKY/6eHtwwarRsDj5vRvigRfR+2Q4i4vDXv/hM8qQ//dG3mFv6LAAcdg1CguKdhm6z8cT+3UKg4Go5YhwtKWwB0IWrTikqQn9ELucZs5NOda9Q7QEVeCHSKru32KxcBM5GuNMkfxkxXaTjaRLKSON9UII1+iZKXl33tIhKf8e8Ec9G7nxMhuYBLtcxQGloIE5ATWb/Bnv8OT0eH19yuQD1fUoRTErOFWaR0lnJCGJgSPzBpPzDbH1XQauCxAIgoEJ70WHG5CHLMoYAcPdulXedgz6W/JtUIcfmoFSEqTK7Z9qWkUf+nrUeuz2THWzAfMmlJTqg72qJLurb/B23iKSgkqLHnZFBSrMdn1bTy5BKGrQrd628I02lnVVefMezQOQKUPmIfd3KLrl7HGWnGs8mbHYzyd1ZG4TUcbuc4XW7FsLX7tBK1HB1uwJesmnzPABzlwkuo5YcdrQRaumEfq9AylIA0L7LtmpJm/SIdJH2Pic0Oafcjj1kPdJplXWSZ8sBSkVsv6ZNQKwDVs+20ufzZsTb62QtElGzgXl/00IfcqRWva6z4ozO9X1h7lHd2sJgvIheFKAHoC7cc8SebsybANvRfYkJ9TxiSopBVYQC0POCjjrQogVg/usK8zd5MpwWGpvv8WL5ZS/lYfP5gFU74M0V3+dti/6TnKsOAdB1iq6zJrFbZPP1aCJCk2sXy3yc8kyQSZskVRKOyOw6mWjD57IoP8+5YthAuegTlktsHOYtDfD7/QxuMpjN8oyc9HeVjfjkKeU/N0OL19sVbq64dMsp4T/Y5Qh30yImLk2diW1usIgHQzXeYKJPEWXKouEB6NwHesyLRr0FECEb5HIDm/oLoEWJpAn5cxcOKpWJpVI53tRonxfHaPLiiMgLb0LQFI7yrb7JcwiKDU3pRAMwKpf0ygeF6v54LhKp0yqP95hKaEoyFX/O9DrX6W7yhiP1Y2ioZvdDbw851Yf20B7aQ3toD+231D7KSHUaLLS2aM47PDujvGBjHLZjIxDSbt9i4nyZqj2J5YMYdWZQeWdmIyLnRo0NqGsnakvDvsqqMgZZmYevIeXZjA6yszY2AF/P0b7PkG8W4VYwHaD2Cf/J+Sc9ZajGDBShpYj6iA7PlH3ZSerCzNgpydeEziKcOTjWGRWWMQBsLPwKmJ3myCAxL7uhxrZrMSRj702Nmg3P270SgX4XNNS2FmWnOPeIia28tdA2CFQ8dRVmry3fW97V+wZHwgd0IP5/oHyzSv64BVQWbZAIxU0G06bOYhCNx/aacO6tnkOZiNWaYPqmcri+IcqsvbUUKSTofJ7VMVRARi8GjenUS3mQGZTAp3rgKDTpq65zDktPWYM1ReCpSN+tgijYKBWBq0ZKUqLREkFM64hwDYxc0tOf5yiw+XWLiXNi+6pF3ymJmIdHPjM2PfVb4Ag9NDqLVTQhb8t1RESOxOCViFDoSJyEFOHSdfK9NIGipjRWCxQIHv8ve2/Sa9uWpQd9s1jVrk59q1dFxIvIjAxnOtMgY1sWMkqEQEIkQkggJNyjQQNZMqKFEBKijf8KTQS0XCBwAuGMsCLjZcSLF6+69al2tfYqZkFjjDnm2kkhGtHwjXum9PTOuWfvtdeea605x/jGN74v02VNRLHoxDRhXg84sLGCcxraBFgWEth1FcIt++qWAS+vKQMPo6ZnJ7VIPVuD0VjcXi8pG073oImZkQ9QC1kSfx+y0AJiFkiIhiDNY5UjfpkCtM/fE8ioRLkGxomubpx+FjOK88RNsmQ/mVOFzKTl8s40i81CGcivT4fkn7VTR9dI+cw8Nl2+FAAhP0lQfzpiSXV8yd6dPvJ1FeToQPB3gsCTSXkI/7dDvnPjvdxUf/DRa9g5SeSdlISxfbM5OYIqvdcol4mYAIB7rEZdILbmaIE2vKn63qAPCrhm8f0AId+EKsAsclvFybJFwbJtN/cLEXQfns8xu8l1E0TIw5JgtAR/Rg0xslYeRw+fCnnj91PCh6LXykI+j1mFyNAmAFDbiZvnRTOUGZJyq4Bm2aPdcQ1r2WGzJWzSdwZ6YzPUfeJk8RhMtgZLMJNhRSivgPKW21U+GAk65/aS4m0hm824youACkB/4WVRKzZa4CqMXCOaoONJkD5WQVqpQtDwc52v2V7nRVID4Vk2sb/fNoibDOdP7c3ihJiDqMQQ3S8C2eyleTRAMeQ5CEW2zXPNZOFyEcMpv67nPs7U+tArBBZ3R2toPX2SbEcA9Q2XFTqF7hI4PPEyPwmG1h6yOfoaiMUkuPIKka+fmTvYwmFoucdUBVKrAuh+G5Oko0GoA1SyH2yLDHmXVENLrUNHFl8qAmXA4pR7HrtCgtn62xL9FeRaTF2YDkMBy8/MfNXCB431liZy3Be5RDJzCCzRqTqDOCGUfXp2LbZ76KnfRWRA3aRnO7WoJ4u0OKnVp39AfsbSNazuJnaDigIOFh7DuIxCQHRNnnuA2lrGFCjGbMpBjaSAShBtyOcWM4oLl4zoJ4RGabkzoPmfBNmynthINnxpvQlRaululgMEX0UyMZBabJTNUO/N0fqiRiXvC+a4d9lMSEm+xHHw8A6PB/j3YTyMh/EwHsbD+A2N9zJT9VFDRY2vX53ja2YTYumAdYG4pLBteb7HMGSspGD462R5wM3NApGzKNgootkiXD2BVgMfTxVZQKIoPNquQsdsQqUAsOhE85oo+bnROkfFkY/tyhyBSpSpIEQlX7JNU4o440QIImlyJn1PG4WcZPYamjMPsfTiCLS81zmaVkD3ak5QD4DNYSEs2LJVGM+CfG9iSvJ8HGUoBDdrlkqyb4yco31bQKGQl/o6wi3os4q1ht3ljKV+bYTa72bH1nVTvdK+ibDPKDttTMDIeFu3KwGnYThrM0OOst2pg9VRYGg/GpRJEadTR8SoMFEtN92kTeGgjzSJoSOiTngbM5f5VzPk+Q4F5Pr5GsCQYcBoIIIDalSIlwMiM37V3mBkr9hwSxl+Qg1sq8TH1JkoIfW4DNCjyo3/SiFwS43fW3hl871UhknmNIEyZ4E8hrcJps/m4KGO0KcDVnPKECOAnoUU5k2Pq/keDfvqfnV/hi2r8/RXHoqJg0Xp2VKN0YvSYZdaY5YtVnWPlo/pvYaeZXg8aCbANaR7ndjR//zVUxxeEpxvDpoIcimZ1pOSieOMMxG+Fpmglf4OELFKTWB7FTMJEBoYljELeIzZ+CBMLBbLPYm7CEtWWcm6lVNHVoHRTJJ3lRM93WuEKitfTYfu1JEgQzgJ8FxOsmsDM+RzDgai5RzrgJBUmRgmnrKepYWmiEcEShVxpB8uJaoDjkVoKnqxn2LM7+h4LzfVL375BLqpUd6arCC0LwEN+DOWY/PZODyxXAFgvW2gbkvYdHN0ViBH19AiHlI9qg5Ynu/leKkndrtrEF9XAN+wcAoVt66EAoAG+vMsaJ4Wu1jQzZsW6OLewKXNEhkWSixENWHJaumlpU1q5OOjDLJAB4vsiKOjPGQAwcSeVY10S1CfPMU2ihRZnLRK0HEmG3rIMFQKBoRBWGV2sd0rICoMaQ5OByQtuMEa6BTspM8X9mLGewPLtCU5PHXRY1bTlxm9wT6xZQOgGg+fCL73eQNJi9mGnUvM81rmUQVydkmtEGZjZREeHw8wzBwNnUU8GIFaq2sti66v6Z6RFokACYzsLvfYAkB/EWA/onvp8bLFq+cEW9ptAX9dIrJaUfm4xcD1bESqcRmuh7t5hvB1VCIlV782x5915aTmrFQUjgAAqjO6yaY6rS2uC7nnfBPI/xfA/KLFByfZEWbd16i5X9kHhW/uTqUGHzYFVGKgnw4IDMuPAOzjbP7Z9wVOlhwkFQ6HsRA5xqJ04mqkdRT5y74tEJ0W1r9zdQ4WArNdr/iGXxeixqUiPVPpXlVNlLkDIIGK3WrYVkkv6hQyhWOmPdfuQ6nEj1QPWVwfkeF+roNrl69TepYEZp4qKk1Yu2oEoBV8YohHgmUBErU3Myetg9FpuYYUfOcNMtYhS636iUlBb8gbdjIHUs9NG2q6RYqYDSo8MuSruDUnxZcjf5cJI/hdHe/lpmo6DQ19lA24laeLnxK0oGF4E7TWY2Ci0thZorinh/ssSLuNGomEYrg1xD3z6JiAdHG6k8VpqyjKSwSeaSajQASAVGvUQ94QfUWiCEnmLlTZ/koNmeQRSmoYt+x0oYdct/MN09/TYrLL4a7yucaZSAVDIkINKn/Pqx7oDcytzX/jZ8HNIm1UE43Z3C+Qr0Eogaii1K3GEy+yatQHl88x7AtxrLEbkzOGZUol+PAK2Zi9iBSxpyxcRxRJ6KOt8vevPYwNcCoFNRHhNNlTBbjOovqGrmGxy8GAOUT4RklAFeoogh3oDXzKEkYNs8+1zKl83zinbFFIO6desj+7zZv0sIpovrfB7129BgB8fns5eZ1CXAHgxa8osmOIW3kUqx5hxZna13XuJzTZZDqR16ZtNJFri4SOTAg2IUsFkpF8zkiiiQjJUtBElEzEK63D17dn6Nu02U83aQVlI8BWYSgDYgoGv62hk8H6R3tYG9AmRGfRI3C/6e1uhr4rJIO2hcdyRqv3tq0kyIheQ5c+Z1xjrguGIiLMAtQ9vba6NTlIjfQMpQAqlMfkoSRgYPcUCA1n+W9yDU8jIU4JlWiVbKS2xXH2qYH6Jm+yqbaeSHGpLqkC0H7AZMfKiw5z7A104+R+VwooLunnvivgnZbebzVomF16VsE97pOAYdKTL8F3EQmlknswt5Ypz89vWisma6ptiWgJ5GdA6q0F//xb0Kf6UFN9GA/jYTyMh/EwfkPjvcxU/RmZlCuVGaHoDTWWpxrcvoSyGQoWCJhF2scTDs3qIGLgADAuncjrxUEjFPS3bihwf8PtGG8KaBvFOcYcckTrOQrWOWmQmmqoKdJNDF2EiYdiAcnS7EHBthOG73KieKLpv+TDqmJu0g9lRFimQp6CbrV4xYYywtyxo4SxMC4noH4WjyJTc9C5VaaICFKoyvWnWFA2nmj5KqiJSg1FzAK3HTSgcubUX6awGNx2Qb/qISslpYbz5ARkCzJMAECCHIndugi4ON1hXVAa4pcK7NGMYV/Cvi1QUtcVTAeYIRU2gfYsClRmDlqyc3fmYLmm53oDbIy0DqgAjJz9+0UAFg7zEwrfh8EifDmX75Yyk/HxiE9WWzjOzDa7BsVN6r0hSHDGWWHfW/g7+p7lvYZ+XqBidUXt8jH7iyiG39BAnLucQfYTiFeDIOmUqeoIfUYpXAwEbwNAHBnJSFnOqMSbdLiriXBQJHw8Py+6JbjQJahSRyTKvDt3qJipO/YW/drCzOmzlYIw9Z3TCK2VOmpROtxvqC5rbFabsrMB3mtEVity1zX8pE6NmCFeV0dEvhTVHSExUifvFTzf78XaZOhWU/0wQeDFHiILGUq6n1NmrEd1VP+XZ6nGkfB+MsCgL02ZYMds7jh3IquqTUAME2F/APMmC9l0qZRlAmJUgqSYnRa0yJcg04OEmnVGeBMwETExu02kVp+ELkxKB2IIkEQe+qwIZobJ95pIo6bfozoCMd7Z8V5uqovTA8wsoOsKjKwEhMof9aUpE8WJJgLwr+gh1QHwS0+kDNDDrXnhNiZg2JYwL2hRC2VE5BtvfVOKMhsUPbwCc/lMqY8XA/SbCirR12vALfMmotf2SGczThwginU2S6bCPx+jzHW7UNDDnW5uN4sorg5y/t3LvKhHG4WeDx1zz60GMOR+RTXRFkXkOuEsrxiKF96pEo3mXj8JHsZ8jFDGo946PUI+C0WE3ep8DJuhODfPPcNqTzBxcgJyvshPrInQrJV7uqIdJykSlddGLNGsotaTKUFIWnsW1LIkJKCdlvN1XsHt6L4yW95QU/BTAuMlfen5RYvSOuxarvG9nqHe8vEOuf+2WvZwQeOf/epjOkhvEJ5yfTiStGXHEKe/L0X/WEVa4FOgMS6jlAFCFXIgFBXUzuYWjEmQGBt/fO0VYLke6pwROy/VRBgTmEwEYFfk4y8c6b5O5j8FaOagMFx6gK8bgsL8MdWOm3LEhlXHYlDQjRNFq6YchZgUgoZdjPD82V1bSh0VgJAMh77AcnGQ4ORwoeBvuJ/VkKViKqf4eYBNEqABRxq4xVajuk3BJnJbC+grWu5ucnUm6aSWl2RrpweCfQHA9BG+4s9tWON4QhaaOsWEhmQpAWC+6mA42O/6IkO1XiHA4J5tFnXlUbCj1LCuqIeYbSIVJsSkhiBdfZhA/2lMAiEo6uGV+Yk4KoMgZEjcHJTMR1SA4ihGaYXhJPdlJzcef6wA+06OB/j3YTyMh/EwHsbD+A2N9zJTNSrCqEjqLIktGrQw4gDAlB4+CQJM3TeqQBkVv7Y66wRC7m8amL1G/PggnyOqTK3JBI/IqkccybuJFq2+K2H6rA3qFl6g2sQETExbfZ67p+PrGtV1ovEdR9ZhDnkPNBBVQHFG771YHnBzTd6neF5CzXNUrNwk0p6QF1RrKCwPCa6Nk2yO9VrTywd9JOydoKlQR+hDbg/QgxL3k1AFQEcUnM2oMUPgymcd41AQzJSExNXZkMlaQRHxKRkktFrUXMrzAy5WlA3tugrdzy6w4Eb8aXSeWJ8py3clEIpM0glFlGjddEBk3NhM/FrTnCf4d/8dj9k5pSizakBpPNZrQkHKtUK5Tt8NGL9LIf6qGvHlrx/JfDeXrUCffV+QgAUjIsXaYGQIPzmOuJMptMrnZKIwtDGoI+IJAMRiglBE5EzFRFjOAqvSiRIYwNAiZ90oIzQ7/9jCI5YOjl1fcF/KvA2nAWrmpEWtOOlRWrrA+64U8pG2EUWZyTe9M3DJYcZp0tzmrNkPRtyVosupz+XZFjEqbO5YqGRTwCT/zkBIQ2Kq607nthAPmHGCiI8T7d4Aua+iIaiz2DC8XyshLYU6QA9ansvyTqG653JNF7H9hGHnJpJuN98vromIhrPYE2Jla/6eISiMI60vReERGRofNyVB+CI4EjEkOLwl0/l0ztLyAyoJhVkQxCLqfI8or/I9ERWKG/uX2o8SYkP/L5J/6x4ieEHm6PTv3WWEm0WZfzeL8HOPcJjUvd7R8V5uqv1oYEYL7zNkp1SErUeMzC6Mb0uYiaJKugnNZY/FvJNFbb2ZiRqPcUTJT1uzu6kxZyhuXES4iTFw/yiQywT4oeU+T80uFLJx9nry4AOxiage06J8vtyL2Ht1qyftLwQbJ0jGz0OWSKw8qmZEv6Fzbn86x4qdzkg0n183KigogVoBLRssbSCQ3tFoorSuxIrgKWEXTuq+UwFt9LShijlzExGZdYsImOsyqyHpzIA2h9z/6GeBIGCWnTsqx5gI87RF5PabMAOWJ1lu8HpN0Fj4co76Ltdio83omXhWAAAgAElEQVQtQNNeRYACDJuUqTQpFKVgYyox6OfZacXsNMp7JczRWAbM2BFnVox4cXOC8nOGntd5sd79zoiaHYvurxcwawvPtcwQFFSS6nre0ObP7Su+yQGNHohVLv2KE5hejyobJPAGK4L3ZTgyllYHk495MmRlKvACDlA0UtIGKb/zKApiwetrdvi50+gv+d6ZO2Aw4gQ0OoP7Ozar11E21XiwiAuI3WDnrDDrtYl4crFGm2QLg5Yec2sDnp7Qe653c9y/XAmTXPl8b+rArTEc6AYAiqH4qTQmQDD6tM4pJgg9McTTJtg+jXDnXAPeW8y+1QKFlusM8W4/yc/W7CUziFnzPxYQBjvhxzG3wesIw2bs82rAgaHg8VBQfV9KK/bo4fDN5Dn3mc0dlo5sJ5Oxemty4GWjXFOzoTacaTtMknuMGig32QxiCo/7YsJ8PyG1tph4FEWkz5i26byj473cVK0NMNajsF5soPa3JfTGomxTbWNS16gDZleU2ZzOD1i3jdSw4n2Z62VLD7UpUHDPqR6ISg8A4+UoN5etHeyk/8+1FvaGj6cBdzlCb7jm1Km8WJ94qHnuwbvdzuFZvMI1EVHlTVqPOcOKM5+dLg4W+KrCYpciRGD9Pd7MbBBZRdUXsHstD4ebRenN1QMvLKn520QI40NHctNIz4bP/XhSdEKa1wifzqsOANdy7NYg6ihCFVFlCbhpO5DdaerbW6cMyCJw/fn82RpN4WShjVFhya4mWkVcd5SdV60ij1DOhGnBmTzYKmfputOpmwmmY2JHImRNFgRdO/FZra4VoIHDxzTHZ483olH79YsLmFdl1vi1wP736RdTBHTcT2mvCyKK3XN9eG3zRqnjkZxkVEEIU+qgKDNYJJYUxNosOC1EE6qJq7yg2ZjJdp7q1iJpuCuk66EoHSq2XwtBIQYtnqNpngFgcz1H9aIgwg8oS5EauQYuH62lPjr0hXAbooJYJtrTAY/PN/LMrPeNEJCqgmrOKdCNUcl9XFqHz796TB+1ybq/AGVits+fFe2kLaoOAPvSuoY3g3TrRgXhLZZRsspiR964IwM/oYq0MYHdlUZIUHP/w7xRN6+BastBO7e/Be5TjZOWK0S6fjNuFzqft7J+AcA+ZtQATkkdPNooPbd+4ZnwlJ9J2bSDArbFUU97TIGXitLyVmz0UZAx9Sc2rOc79V4emaYBFYUop7yC2SuZU+UBNWoJzt/l8e5/g4fxMB7Gw3gYD+NfkPFeZqqV9TDWY9+VONyxsXRroEeF4SlnbRO3htVZi1nF0m9RobQOS47QF49v8GpDoel+3SD67N4QVpNsbtQZWvIU1Uu9aCLlNzwZofZGqP32B9usaMOQ1mHP8oYaQGLdVZnGXuwUgpk42Gwy/BOrgP7Ko2eDar0YUXLEP3YWis20TasAFcWYnOT0OOteUqN8yhjNJisG+ZDhWOCY8RvzIah9pw7yPrMxkpG7pSfhhkw4nagy5fpkKBgW5tf5JmD1mLBsoyO2XYWBWwlO5gcEnsdvvrxE/bzI36UKomREB+bzZRg0uQLpIdfBtSL4VK5vESS7C61F821uedl9z8EwLHroS9y9ofvF3hSwhwxB7j91UtfU39QoODt3NUG4CbKLydkFINh21IhlUtHQsBvOsFaB1KjWfL+MKp9vwFGNH4paZOgYCjFNSKCLJsbVJkKbrLYUmK6pTUDTdDLHw2jRXdOzVb+0cIuI3Sc8P3XA44/J43RZ9Xi9XVApBkAz63FQdL5+X4h70GLe4TBarDd0Q4ZRo1nSM5gQp8sZlUXe7ufYbwhn7K6bI7EKPeE2qDhh3Vd0P6faYESU1i2z10elAJKoTFkgRCTC1yT/OK4yrF5dJ4EXENzJt8XsuRLDcu1iVhhzEfaQpQ6Hs5hbXCoPpaMwfrWK+GB5BwDYjRX2jMqovaG2l/Q9R4VwMpFLBRBTRjhqyUCFG5ES4ypn7nZjRHRC1JuE0a7kTXpUR5C4r47b+dJ7kstNnJYgioAY89rxro73clO9u59BDzXCwWYj39MBl4/WWJS0Yu+GEj0vyOvNDB3DU58+usb3Lm/wP3z2IwDAq9ZmpaED3cwCFzqV609eIfCqFbuSbmCdb6gE95SvSa81fsg6qVGhu+NCRKCNruAWnqYZ0FluK7iZ55t+ANRE87X63gbLhp7SZdUjRIXO0Xe73zdob7hdaG+OzJKDxqR/LmTo1tIiL+bDB5XrxSUrTE10QuWhUjjSC1WDyvWWWcjwWhGAXotkohrVkTxgIuLgqke8LXM/69lwZHTRD1ZINYehwOYVbWbl20zm8XOC1MEwum01xvPU6wTgYKRX189yG8poNNWPWclotuxxYCeX6kUhsHn7YUBz2cIywabd1wLjFjsiXXVXvFrXHuaWN/szh7jMNdTQZyUpOAXF7RP6oFiFh8lsXdZoVoOCfl7nPuTkEgOQFWFqe3IqK2CBNhbFRKKouZaWoiFvMBo+R5WDtUJHtF2JoU3EPEvzCmD21+/xeLHDbmA1JDvi7Z42x1f9klpz+Dh9V1D5AMDqaodPzmjTeL5ZkbxnqjIUAd+9oI35zX6BbrS4Bd3H+0OFKIGQPuqj1H2uJftmEpwEcDtJKqHYrO+raFOcEuRkTm0OGqUVRmREtaiaqb9UaiaklgNRlQlDrqE6+LialBXSRt8ZFLMRCw7wr5odHD+gv747R5/kO1cjcJ+h4HDihGUVPaktpboyioCC6779usbUJN4ctMDGpps8qw235U32P+E8dEB/FoWDgqhkE9ZjbikaTgLC3Msxde0QdsVRz/+7Ot79b/AwHsbDeBgP42H8CzLey0zVmABtAurzFpahrEXd4zAUuN9RtKtUxIKzu6vzrRBeXm5W+PlnH6JkWGc8C1AzFn+YDxh3JQLDRtGrDOfVDmDB79hpqEFDXTBxZl2LMo87CVCNl8iyWzc59CkDbO0EYtt9cYLynokcE0Ht9lmAvxihGeqpy1He86tvr0RPFSAoekpokGgUzOpLDjmNFyp/dBpx1EKPH2pkuLwMQGeyjm+VM9BYZJKRGinyFfGKSTuHnhJx+LyS4bJfjliwabgCsOkMfEmfVddOYMTNUGPsLRSnCPubGYpbJkK1Waw+1IrIWykDvSL4HWD2rCaFJACw81GIOLvbGbUt8Gl2XQGwW5HylKECgH1K7S/bW8rMyheFiEv4Cug+zE4s2BbZh7IKcFvOnu8tGbhP3EoEotb0eeXt5JzFA5MyhgQbQ+esR3EJAgChK0U4EnhQ20xiCpOMTs+ciCkoRcpjAKsmFbkkcPHJHf74g18AAA6+wOfbK6wqSt+v2zkqFpCwJiAWDucNQbcfL+7wVxbPAQA/232An1w/AwC0HbPyOaOuFj2+uqN+lVXTYTYfcdeyn2pnc1lET859b4hlXmQERJCqRNRK9+rkGYkmwu5yS5ZvMhol0DIYOrYBBQtD2IOCm/Hhi3hkkjBt1RpOoqALxUYhFpmcRAQ4Vu1qPE4XrRDAvtqcCZrWD1ZajKLTRMhKghout9dARWgTUfKa1R1K9OyLLOWcNgs3TH2YhZwUFUKZSUflOovJ9BckGJMyXO0ybK6dwsDtb2FGXQILZuTvnq9ojrtpHebdHO/lpjq2BXQkM+TlJbF6X9+uEINC5PpTLIOwbIlNyItu0EARMFzQseqrAwpeINq2QrXqRfWl77PIdwhanEvOH6+x3jXwX9NC6y5HzPjmqssR9/dzhKRMU0SoxLp7W0CFMj9wmtxLANCCeEI763JF9cOe+wLX25nILM5XHUJQaNfc49FrWXSiiSg2U1WmKH2lRTPKYjqrRtzezRF40YeGqLxgU9CGmjbqAnIMIC9Cuqd2FJMYqIBsnND0nrDKDhlJmrFqRkFBd9saGLSoI3WbKrcABAWzGjKD9s5K3crNcz+i8gp2IngfTRbqD03E8oMNHG/UVeFyfTsomFbDa27ZcQqWYejuqcfFJwRbruoOd22DA2/U9Y0SNmT3yYD56UFq5OZkxNkJ3Y/3mxnUi9SrwS4mHMSESfBT3mmU6wxBDieZtS5liCRrybVTAAgzD8t9pH7UFGgli7tuAn16ZmgmCPKmwsi2LKEOUoOkqprGkltj/sbjr7BkDPyr9hzfX77F7UBf/Lqdw3Et1hqP2josS3qtixr/2/33AAB/cf0I3ZDs3BS816gXFIg+Wu2wKOnn+67BXdtI/ZxOjv9fBGG+h0WE6s0R81VNgrmgIkLSqDQxtx8NCuMq5H7UcmJk0Wcz88Cm9X4ivZk2m2JLUH9qrRqXkUoqoGAzBUXR0gacWuBgogSzAHCznlN/PWhd0pNnK3DAoZN0YWJ6qwikW0lHBK9weLFIX1o2Xz0jCDb1ENsW+X4xk+dTAc3rDOu6GnDpcJbkRwXydQo6sdtLmjsAUDMHWzrsvqHeofqNofn5LXCpeYB/H8bDeBgP42E8jN/QeC8zVXNfQHcFQhXQfk7iCdESVJngVOU1hlf5b+ITOYtQlwPKFYVfH5/fyXGHeYvOWdwyQ3HYlxL9n19usZwwiNvC4wd//UsAxOI7OIrI110NbaL0EDavck9YdxXw8e+/wFVDDNc/f/MEhrOocTRYLSjbbcoR67YRBZu4t/DJlNgb+NZK/5x2yD2laiJ8YEgkv76kYy6aHusdEaauNzXQGQnJzGoQckmwDDemcM3naB2OiEBA4r5k8kMwUYg0OpGUVM5csWIyRVug6+k8VBmgl6OIuiMCxZLm+HTVwuiAG0Uh9OgVAlubhb2VzNe0pJyT5tg1EfoZfecPL9YwOuB6x9fTWUEsoCP8ystxlFMYL+j4V8/usazYeqyvcPdqhWf/iN62/QgYf0RQZ2U9Dm2JmqG40/lBhM/DmxpxnqF31WmYLglqQMhZvok4VJko4ksIiSYpAmWCSRC4XdVevst81aEwHvf3LLpwMBNEQaF4azORZh6yOpeNiMxgVbXHfNnjR2xP91F9i8/2T2jevMGbfolvtqd8LTT6pARkNN7crHC7IJz0fN7iOQuaKAVBeowNKMsBlwvK5CvjBP7tewutI8bJ/S4sXhUyUhJZFSjpNW+zqEUsWMAksV13Wli9AJV58nGQM2GVjR/sqDD67E0biowaDCu+v9Mzs9PioazcsWF5qGJGXIpJ+SQAMMBhy3rIbtLnWQRUDRsdRNZl5u9ikhISgPG+htnrrDzWZAJi2BUor42oevlqwo6eeJ/OXyjoMcLN6B/GVe5T1cn+kol6yk84bjWyz/LBAC8qzCcWd+OSyk7v+ngvN9VwOQCNhq1HqSv5XUH1IhbzHobMSHS9FZF8BKrBDQx9viydiArsuooYppPN4PEH9wCA3zt/jZOCFX20w1ftOf7imvpa6nLEzT3jJ1/OUG6U1FvaH/b4Oz+k2tQPZm+w9g1+fPsRAMDogP1dkuoB3IxhYh1QWod9oIdPDxqKNzPcFdQsoSc1obRW2JgN1quAYj5g4IXq+rqZMCX5YebXah0QebWIM4c4mKzpZicwWq8lONGeWMEueaJqSB1W9wpqNPDTTZXnO9oAzaxSrSPcrpCGcX1GalcAsDtUxCRlSOzygzX2HWFgh22GuPSgCNZK1/rRgD94RhvDwvb47OaRsCqbarLKgtozZFz1WC7os0vj8e0NbyBvGpz9uRbozM0An9xsegP0BgNfi7WC+PaGlRPYb7nosLmei/oMQm7t8Quqhdq7iXF7ukwlSUZOW5zSKOtRPEcBgptjm9zSI+I8tb8ohCrLLsYqZEWiyWKtbcSPHr3C3zz9AgDwcjhFyXi7VhGDN7hq6NlanRywsvTZ/+urT1A3g4gYfPXyQsTwqyZvoidlh+vDXMTwt/0MIfEVdMQ4WCndaJfbj1SdXWrcYMhrdFLvm6pgAcjeohMaua/5OVG5pSTBv9MNJBSgdpIqb77yc1CwO4OC5TBtl+X7/CxOavzMME/Bz0QeNAaF8a6W5wQacm3rkwEj36d+1IhOy76sdkaC2ZJFYdJmCQVpa2teUpCRBGt8nZXkik32f40GcDaXMaLJ7kdmON5IgWwMMS6i/Hv5ihjyEszO+H4Nkze+o+O93FQXJ+RS82S5xTd3tPhdfLhDbR0KVrs5q1qse9qwfvH509w6sFOobq3ceH1fSI1vv6+BCNQX9JRdLPdS9zn4At/u6bO+ensGN1hxsDnS1zt3wHd7fHBB4eJ3lrd4y24TP799fHSOh0MJ06QNJuC0oc/dDyXubhZQbSKbIEejFWl7pgfTjJAFZFxEIXiYxmHcVFnhpJ6oMgV1tOiIpitAi9Z0s5ks8nrIrTdh5qimNTGMFklAxa0uqS8zKMkSZpet1LoPbYX6rBNCWWUd7pms0ncFbOFxdUnzqFXEzS27dhzyJiH6wec0p3/t06/huED52c0jDM5ixRu18xq7fdJZo5pwqn0ZnclIu91KFt3znymU24ibv8qkrqsxLziDhnLZ0cYWXoy3MWgxnd7sFzA7I3VfXwGOs+L6pMc4WFGkU72WbFTPHfUjpjluvJC8rnizAoBvrk/hB4PihIlzJm+WVelEASqNhjkEr29X8Hx//OjDl3hSb/BldwkAuO4X2I1ce4XCwvb4zuwGALAwPf7h2x8AAJbVgF1bY3zLwaFXUI/oHB8td1gWdE6lcZgXgyA6BROcAGB9mCHeVCKD6Wchk3TiRErRROg2W7WFYlInTPdizAhACgDTkF7SMdcTlc8bQygI3RHnolZB3/LhmVCWpESpDjnZfCcIEcqQr1np5X4f+4L+xhyCqnISWBy2Ez6B4v9EdjLCMafCtMf3LXREcc3oSElIjZ/Ijyb5VLtX4qoTyuP+03EZssSlV/Jd6RiQOSXrxISA0MabZAvTcziVBX1Xx0NN9WE8jIfxMB7Gw/gNjfcyU03jq5szYeru+xK7rkJTUhR4f2hw2lDE/Ls/eIEtN67f7WbougLxllVf3jYY6uQmrYEyEAsVwPN1dexDyHCZriljGJs8/Uko/KTs4KKWFpj/5dffoxYBAPViwPL8Hjd7qj81zSD0+vN5i5EZlbf3c4KpmQXojRaRAqUj9Ks6e4YegGGZoaAEC/tdAUSCWwGQ4lSq4506eu2Q6qNKskqAI3IWolCTLDMWuTZt7yyJXkz0le0mMSBJq1SYtjrCnuS6Y0rsl3UPo6IoyfTOYn9PGU/RjHh2vkbJ1MyX2yWmrjqeo2w9KPgmYMFKTD97+RTDjhWIioBHlxuUnKndDQ3cmOpgivSKeX6Kz2ZIBja+ipi9pF9Oft3j2z8u4T5h9R8TBN7Uix5dWwpE3d00sOyJa1WG3gDKjpLYvp/nTKZ/PYPZaajU+lTmFpJwsNA7g7Cg81+sDni8pO95VrV4vmPOQNCYcV0VAPYT55nRGSzrHs8WlPHf9TNREHOtxdkjYvv+8eVnGKPBP76hDPTmMMOioGs2L3oMweCfvPkUANVY0/29bWsMd7X4kw6nQdqivn5zLplYMxuwbDq0PbPzARHNj14hzjw8GxbHIgDJW9RN2lgc1fczOzfDlPaexDV8OckYY878dKdEXOUo45wBwxlngb1C8zK3WYUiZ2ymB7ON0wXNh4fOWR8aT9eWTywMBiEZMSvSDU9tXTEqdLuEIaucqXoFmCwSM25KQZz8LMBedKTsApB2OL+tv/I0dxNGdIK2p1Bt1Jzli/vMpK3I05oiqmflZA58VopKTj+53YYY7r8N7N/3clPVKkKzoH3LVlVh1FA6ilB+XY/SSrGse/ydx58DAPxjjR/ffoQvIsFcuK5g2iSGHxEGnS2SzEQ4/HTEbJFrWCFonC4IT9m0tSgcvdlcomtLocejN+L8cbHc49vbUzlGjJC62LwY8OWOiBtaR6jGiYqP1lE2g3FdoWoVCnamIflBnpdOiatLLLkXbwLJxscs9p6o+trIeSRXmlRHSiSZOOn9Q0CuZbKSkBhZO5UdPVqDYqPhGUarP92INNvb+wUWs3weSkVpz0jwO0BtU6/XS5mDwuQ+W1wEeFbc8YuAohmzucFopFb4ydMbVMbh85dU+/bbIl/PpYPaWZQvGI7cA/05b9QTKPvtH1YYHo0oGMIrKyd1QqUing+nQsaBy2o/alRZKYq5JKmXWY0KJgU0NsKdOeptTpeS689q4NoiL9C7+5lsRDf1TIIwpQO1WY0cKDojhtfNxR7PFmuBcq93c7R73t0V8PtXLwEAb4YV/tHr72PL/aSl9TDcl/rV+gyl9XKdBmfkuXM99TT3z9g0/monHIW7tsGeW78ObUmuNKntp7USPNjKw3UGcT6BqZPCT6tzjbmKiCGTupTLkC5AG4WIy8tkUrlAhbzxaa3Ajyv6Cw/D1oyzl1QWSQ4ziFnSbyqvCfDPLLzvVj7zFXpDz1x6LkwmKtlyhDFRriHZ3dH5+s7kzbAKMLXHyO1kutPyvc6erXHoS4wiR6oxnrEc43KELRyGNxS0262WQCJUkLVgXEa4eZT5sK0Wyzd74A2T0fxg8jwCE1g4LRfJ+epA8LLKS+Q7Ox7g34fxMB7Gw3gYD+M3NN7LTLUbLIwtKHtLXAGnSVeUw4xDUAC3Ony4uEfFIe2dmyFElX0e59k7NDYetsnwzP7bJWKKfJ1G+82S3+NQn/S4vuPfo8Jb1hYOjjLmBO8Vpx3OV5zRdhWKIrONQ8wh4M+fPxHukLUeMaqssOKVZGJ2TZT/kcnGvmbrMOTGbAAQQ+tJE7pE/KNGnGrRRmSlHmYTJ6/VZIoMMN2eI9FxSaSOdPw4atGzNZ1GKIHyuwQteq9FIOH0dC+ELK0iKuMEAt8vSnz95pzmrSShhulrUxvHMBiAmYvzeYeTpsMb9lctK4fzJWWS82LA8/UJ/IYzP68kG7Klx+qTLW4NIQfDuRLy1/I5UG3oe93+FQXYgHMWdehHi5d3lMrEoHCyPOBuTSfjy4CYzK/VJMMfONtKIbCDwNex9kftFENviQQGwJx5Ur5i1GN+cpAsedNVkvEEb+CiEoKSLTxWpzmbvunm+GBOLPYv3l4gMJRYzEZcM4muUAHzYhCimPMRv+AMP4waZTNKqWUcLXwSN4kK8XSEZRRh31ZiRF4Yj2pOz5LWAVfLPdYHwhzX40zeE9nLNfnyqU7D7nL7kbSJKSbKJGJRGYXRqzz9nkhG0ea2Fu34teInmglOyisppeiBjAfK9QQ2dvS6csuEOG5DaZ8ew/upNQtg6DmtL05PTMkJvUkoAiLEqKGYjVBMfBq2JcJ1Jd6/buWFhLbdNXD3JWkig9nJnCX7TQHsapRJr3jSDhc1Z6tgQ/FZQMGlCnPIxK1ogHHiQAdgwviHCGWMC0LJkjaybZn09VvQU/NebqrDoYBGQQ90gi2DAqp8RcOmAF9vvGxX6DwvQFGjsSNqftgPfSMQ5uOn9whR4eaXJLekPeBToc1NzLRtRHdbC8RTzEbxuYxeoSgdHl3dy7kceDMwKmJZ96TqBKDtS+y4phK9knOqyxFtV8JPFI9kRGL4iTpMMXFaUflhVmVAPBioA8PBS5dl7RLMlAzee3MkPg5Fzh4A161CbiOwDDubQSFqA3eS5ydBTdGQNODMJr9Mhzvm73dDgQ2/ZVV3cEHDM5Z0f6hR1QwhK4KGd1yD2+wa6aU1NqDk9pjdtsbmbobHj6lmeFofMLM0j7fdHPevlrK5LT7c4ONTui5/dPot/uz+Q9wq2lRjHcT7s7kNuP+U22Y+6PDR4zu5Zj6qieKWxfU3p8KAnnp9Iippg1KOpQfTfDtFYuQAqVmZKG06xnqZAzcaxNZK+eDRcof9wG1FfSni9LN5h1k5SlvL4Iy0H33v8gYfzO7x0xuSC9Q64g+/9y0A4Powxxdv6V5/Xp5Aqyj10P26Fj6BmTus5h02zJzubxv5mnY1MOuZJnnsLXaHSv7++IQCq09X13h5WOHNhjfx2mXFoKAATAy1VV7IQ4UjSP3IH7fPovChBN3Xk/7TxC8Ilhit0nvpgMDQrTkoVNyqbgTezZu4yFieavTnub3EL7KMoCkD5sww397PuAc8M4NTQBxaA9XmVj+cZHWxcVsK677aKIyLCHfC641XAgUrR16zUxlGicbLgGi1+DLXtzkoGJfHtU7d62yUAfKABSAM7DSmjF7VT+rIoPkURrEFlAbCbwH7973cVJWOUCYi7KwsVHo+Uh1zyKIAgU1/X9dL3Bd01yzrHquqw7zmJ+g8P/g+aFxv52g+pt/P561kQN4ZeTj83kI5Db3s5XwSGUkXAR9d3MPwnfh2P8d6y3rEoA2ztrRiLCbN8N/enMo5rbcNtWYU+cFMzigqUJ9a2miVn9TuRgV3yqvM3tJDljLViY6xaNAmws3MwXPbgm4zyYKOn3/1Va6pKMfOF0zCmPaK6hEoFgP+pSe0ePuo8GfjBwCA3aaReVQqYneoZEGOUUkNFVAIQWH7diHn3Kxo4RoGmy3/ioDLq61oz7Zjia9vqTZ9uGmgBo3mGUUCf/f7f4rfraiG+D/d/wFlYhyINZ9XaN7SF339r0BWEm0DrrfzTLoCCXUAQLitSN4v9SROHTomWYJKbUnpGFc9TMouOktkliExvoAx1edsBGqPkrPYL19eZKEMr4S8Vi06jF6Lvu7QWVxd0D38uN7if3/1Me5eUXb9t3//l2i5reXN/UKs34a2hLIhy+Z5JdKVs3mHty8yFwA64vQxHf+EkYQksFFVI+YskvLd1S0aQ+f+y/UV7trmqDc1BQVDW9AGnlq3uuxqFKqJBZonkldyV6KeUj5ddkmKk8Al8QKUp80iZaC+hkjvFbssHBIsXSeRibQK3TnkGOMqEskPQHXSyT1Rlk74G9GpI1RIFUFa7yy3t7gLmpO6GcVez26M3C/DCfWDpsAWQYmwSiwiitMO7jr3t0sg3WmUa43yPp9zd8nPls4ZfyhJ8GLqHCUoVsGfzW1LiMj8jQJiCWcOHMOLpCYAn2us7/J4qKk+jIfxMF9SJdYAACAASURBVB7Gw3gYv6HxXmaqaVTnh5w9OgNTBniO2vw8u3Yc7hroi1beN3qD6zcr+f0lv2d4O0O0AR98Qk3ur25XAstBQeBHGZOMa8mC+rNqwLavcr01QGTs/vDJC/xg8QYvOor6v9ydS3P8rB6olQZUF4SP0mKgdhbVDcORTYRbhOx/eFCSIY6nAXrOUmeeI/90/qPK4uknA+rZIDDU/q6RqFg7wK2isPiSkgtAkW6Cv6Lh1hnOGsIsILBUm1k4/NGHz3FeUBb+VXuOnmvOSkUM/HNvqTE+cng7bzqsuN583zbYbBoorkfpIqB7RfMTbUTJJvPfvaJr9XxN7SXb+xkiO2WoXiMuHf6jH/yfAID/ePUTvGUJnifVGr/z9A0+/4ffBUDs3/XvciTfZKF5BDJjqLlVa7trBJbXjtqNUn0LGgI/qol7kB4VMOY6dRwyQxmdhtmb3IJRRWg2Vnh8scbtdi6ZsS09Rs6IypNBRC20iri7X2Q728rhckZz/09ffILuiyX0E5rXH7/4UGqxWk96UtI1vuXUrwhihL29mxEblyHH6rTDwPX+l3crxAjxvf3gZIvHM2ov+3p7jg1nz0ZHFMZjzxJ9tnRwyT+UFb6EmR2VyP65ZWbWKs5m073vZhO1qaAIGk2i9COyA40GoIDuMsrf0jGmQgfa0e/jgu/pMkPCviYJzzRfQ1egmdOcNuWIW66rw+kjt5/YGyj2vXVlyBA3yEBCstfzTGNWg4bZ5OcuFJHM6gHE0QC/niP5BoQiCuJd3RhUd/n7uBmEA4GJNKYKGtFAHGxsm5Wp0hykDFSFzAQG8rGhAT/ViGEpWH98O72T473cVJWOUDqiLB3MZFFouxI+ceVtJDNfAItFJzDVtqvwYn8i79Fri/CG76Amojgf8PqWNlzX5/qHrUeZ7VgEVPUofYGnswMst4zcHxqstw3YzxwX5zv88Jxk80LUWJgOc5uJSj9/9ZiObz0+ekTFnTFoWkx5Ya/f5rvXLSJMlxWFfBNJthFktN1xj2L0yL1+AFRUUNwrmjb5/RtaCOzaSI3WLQNiEWC4ZaV74gQiLWonpJnQWdLPLfNCuOJe0fN5iyFYfNUSdnZ9WGDG0PY+lvC8IG+u51A24oyhymU1oDJZYaqoHE5ZD/luO0PkheXTp9fYcW3xxWaFQ1uJqpEatCz+9tEB/8Hv/Rj/1eVnAICfDQb/5Zf/Hs2VHfCrt5dC0Dg8znVqFRTsxYHPI2IcTZ7X2xI6wdU6kpNJWtViJq+EWaDeSVDgE8rc3mQah3DH7RIDw8Spbn15wJK/8+ubE/jeQB2YsDYqmEcspVg6Wch9Z1EteoFutY742RcEt+u1JQJb0uA1QWrCQ18IiQsKwJhNy83WiG2g7hTVzFgv25iA9ppLGlXA/CTf/+u+FgPzfrQTp6iIfVvlzSb8Jai815NNMFKbCgDVeLnn9JpaRCQ4megAk/1dhIopOMzPiHymKCfl+qoeAcXSeuNMob/IrTfKZVKUm8Uj8tNs0ePpCQUPL+5XwAsKFowCotICg8bVmHkfDA2nUg4UchtamWFi3WZZTPqeEfZbOn7Z0b2UNkFESEuQbbl1i/d3X+XadCwyych0ivp5E0TtgSL1s/YRvlai8GV6iJzhcJbbg5RXQg4DSGvZDnhQVHoYD+NhPIyH8TAeRh7vZabqdwWiL7A92BypapBzC5MrqpMOlysW1/cGL9k5w99WMK1O4kjkMJE0PE8HhKgwY3GCUE8gmKBE8FqpCKMDzucEKY/eSKtAjArjthS3lUfzHQKHyJ/O38Ig4nagKP/57Yk4UXx4uhZPyjftEn1bwF6zF2WdlYvMQSEUEMZvLKK0B+3XNbRNmQEIqkrh+nIUJaDuwCpA/Lt/7AXqg1NQg0Z4RufS1KMITyQ2ahqq19BnnD0+eYuzmubj1X4Fqzxqzjr3Qym+mjFoEU5HQy0XA6MLN86iZBJXYQKenG4FDQAgkOOb7ULakcbRwG1K+S5x5iUC/5vf+TX+5OTH+NbR9fwHr/4t/PmXxILFxqLYanGS8XXMhutFlGzapSzpLZ/zVAxDExNVjMm9ElamPmiBDglSjFDn9A/+vpSWkWKrMJxE+EtOmfcFbneTnoYAUdbCPOKU27Oun5/Arjmzedrhk4s7vNpSyWFzN8vaswGIOrPk211FpQEAsTMTmJsY0Ils5Rche+yeUkaoXtI93l4Y1Od0fxgT0JSjqFatD7X4AAefnVaS52s1Y7eiQwHcMyt+VIgqQ7LiogMAm0Kcp/So2GCc/mT2mdyTsssEaeoRuTwTgXGeMz/b5hYS00M8WH1NGamI49sIz8L1YU6tT4kR3W5qfPE1zbfdK1iXjhERZhOC4BQtCgpmZwSxCFXM5ZneyD3ha4KZ0zNuN+bILxjhmP3bvEowbkQoJwzomNWngsWxni8Axdlp1JShpr+5BhPXp0xw0qOSDDkaQmkKhqntgR2Wfgsy1fdyU9WdgQb1qMqNYoFwMqKoaVFeznpcs4Vb+NUCJvVaJrSImYFuHqEZUqubAU9WW9xyr17nNXpuTfC7PNV65tCUY1ZRul2JeHpROcwvW3z/4hoA8Gy2xtLS8V/1K/z3L/6q1JVmix5/8OilHPd+oM8NUeHsfIdbhmCra4Mi9azNIkKTH1pVeXlwVREyS3XUUCwaD1AgMLJyVDEbYSoHz+c8tlnpJm0UCebqD4UswlgXMMlCb+lRPT7gkwtSHP9gtsZ6pO/1u6evcfAFfnl/RW/bNhjvuba210BqkdARWgcsuI46OIOaRdZL41EZh46Zqj5oPFoQvFwbJ8Lsv3p9SXXkG1bWajz+1u+Teta/e/FnaEOF//SrPwEA/OzzD2Bvs3oWSQemWmYQ2JiCEcjP5mU1gYZxtLCAu0EAhulYqrHY5mN7tsETU3ibj3/4aIRdjhlSDgpFRXPgvcZ84kTjvMbNNS3kqvZQ3Lv40fkGu7EUlSOlI8BtOB4Wusv3QegNVNr4Oy11Nnc1YHnW4pCUkjYl9E0h1xpeIbAiVzK7B4CSDSIObHm339QZIlRRDAZMEaBNkHsuthaK582XAWZrqH7K86N3qec5s9vdLCLUUWQzlUZWs1I4qhumRR4A+keeGMVp42si0km6eZT67dRkAuC6ePo3rxB9ttRDp7PqWFY2pKFpjQCIh5ECUbPTUBHi7BTneR7N21Jah2JFfBBpsblVohw1LojxnDa92UsN2yZWLzvPpIrMpObpmyhBCyIFEyIxOAK+4jpywU5MTZ6TpCSlBwW7z3OjXT6Gm/H1MX9pEt/B8QD/PoyH8TAexsN4GL+h8V5mqmpUUIagNrFcKgO0DWhYDWm9q2E+ox5HM4FLQkGKKSmKc8sgZJiLeYttXwlU2a4bEXWYX7WiX3u52OPrN+dimxU7A7OgUHJWD/jR5WtoDuG+bU9xeyC4tx0KFMbjo8dESLpsdnAccrqg0bNAxaPZFt9uT1Fw1qNCPl/fsO3TxBtVMTwYvQISPBsoG0uvi16LzVyIxAQWsoiKwJgwIfpdstMYJ1lshOeG9NOrHb57diMQr1YRj2rKJA++wBfrS9zv6HuP60o8MMVoGgAcqUal+Z5Vg/TwXu/mOGk6UVsyOuDbe2JNax2ETVyWDt2+FKWkf/uPfor/7tk/obmKEf/JV/8mPntOZtt6YwUaq+40CQskVRwNYeSW9QjHmReuS+qBTKfvM3ToSxDszOhA83WR+/3KKAL6oQBiGaEX9MZwsNLYDxvhDpkgpAuPmoUtQlQorRfYu+9K2CobK5wts1LXZtfIeZye7el3AKH2OHm2Fhj29nqJyNci1gHqij5rVg8IQcMlZvPhmDhULAdRbIpRoWBEYXAW+0MlJYIYFHS6vhFiN6hUhNZRMlWELOKgeo0wC+JBSoL0fAgDgWAjW7OlZzl6JQhU0AQb96dJmCOfv+4IphSrs1UUP1vtcr8mgCPBBxVz5hsa+oc4y/ducJmBr3KVArEIWaysN6RfzMPNA5UoAEGTAEIKBCrWEXAK5R2TAouMjoyrgFBH8Y1VjvppgSyQn7JYKPJRBYD6Oq8hid0rSmkT+zsV6bqIzrECyi3P40AZKUDM6FABrs6ZabF/ENR/Z0f13S3MjAy4p1BUCBobFgtoviwEThnOgyzqxVZhXEWMT9iB46QTWNFHDR80Kl4w+srD2MzwrXjB//rNOdx9rnvp1Yirc2ICPp7t8JPXz+RvT1ZbgcZc0Hi22uCHJ8QGflqu8WYgOO/P7j4U5uubdom335wh6cdPFZRCHRjm5c8uvQg3kHLPBH4pA5pTgp77QwHP6kqmcQhe53ahoPKCxjBaZmkqaGaI6kFBs1fmp+fX+OHytdQ2F6bHr1qCe39xf4Vdl1V1qvMDhgPLOHotgUo5G1CVjlihIHeVsswSd4M3cpz9uhbx8eXigGfnpKD01S+e4O/9a/8j/o/1dwAAK3vAf/PmXwYA/HT9Af78px8Le9qX8agdI1S55Kx6Db2i4ztnSOAcAE74/uL7R/mJaL5TUAeL+i1DnENenMbVpO7FYgCplqw8ENl5xtSOzIV4w6qqUYI3BI3eGTGJCKOGZVixKp3M/VSYAgA225mc/+K8hTUBZzVdt9u4ymLvhYdnZvGgClKxSrXAg4HjDbde9ogRE3Zx3jQK48nkPt0vTiPsMwM/jViQ0pi4stiYVYdGdkJK9etBZ6WkIkqdFKDgObGhi42Wa+HnAXaVFe99Wwk8W66JCRwTL6HLsO50M0QAUCC3RYUsdoAIFCdDNm5orTxrvqHyE0CSgnrmpGtADVqY1wmCDQzrxjJg9pTWnnE0GLesflYAxcYIBDueOehDlja0a41im2FvCd4M8UNS2UGPigTx+edym74/w79JbWmlxOnK9ArlfW7TCxZyjOEqly2iZs/aCaNYucw2fpfHe7mpGh1gdMDICzVAeqf9mxnmX3NUf54f6PqtgeMbtPtoRLEY8Nc+eEHv0wHfbCkDct6gdwZ7fvCb2YArttq63s1xSJZaL2tgFnD2wZrPJ8rD9pNffYTFWYtPz6l/8vowl7rS4+UOj5otFtw89sXhEjc91X3nxYD7jrKLN/cLqIGkygCusST1o8ojjEb0RP2mFM3aUOb+ymgDzh5thQQU+uyC4Q8WykQ5RlTxyMBcT76PXxeyyMQPD/i7v/enAIBHxQbPhzMsmY3zVXeBL7Ykeff2fgE3WMyXtKFTbS3VDPNmPm96XM33eAlqYTp0+Xq6oLFfzzFueSMaNXxH57iJwE/+nf+WXvivA8B/Ie/7k3/8n0n2/xf/9DvQNmJY5Uzb3HHU7RVCiKJUo5cjioLmo7utRc1Kl542CSbtxCJA8aZBzhzHmWt/SccLVa79QVObRKo/+UVAOc8bgKgYgTbIdM26Q4ngFOk0AyiWg5zjSdNhxtZsz9cn5MqU2p2CwtljKn6dNy32Y4nP//mHdM6jgj/jXuZRZ6IMQJlYIjhd9qhE69dA64AhOUDNBlwyulMYj+d3J/IdimbEqHlzmGSctvDUnzxOArmklDRRP5K5S2SkVk8CSsrgUvalMkgAs9NwuhBT7mra52nB5uOQOUjrQTTHvZfK51psLCb3x3yE93qi26uyLninRfNWOYWwy25I035lFek7xKTtXAYJmNQ3TXrEcw2ZW4f03ggHRI/chiWvBUaWXESkzFTXaV5zUDCtr7qGNmK3SO/L5Kz+MqDYKJkDIrrl+ZA1ZEH9w4nYpryCW2QE4F0eDzXVh/EwHsbDeBgP4zc03stMdftqAd3UUI0XAfDxvkKx0UcZaoqwuu93QuW/WLT4welbFBya/fT6mbiMACRofsJtC0054pYNxdtdJbqr+lGPpiHRe4DhwqSdOx9x0nR43BAcfFHtkcAyqwJc1PjFjtw/Ol+g5qKQCxp3zDoeNxUxIyfyJAn2m8169IMVBxtEZHF2ADH5Pzb+GBacwsJeQddePF/VtPUmgvw5+fjmZMTJR5T1/K0nX6HgsP7lcIouFPglf5chGLxeU8jsX80w+3grusaHsZDzdaPCo0uam985e4NCBezYQL5tK+xf1/mUW42G60r9ZcCXfy9npP9v45fXlwg/pfYpG6kGlKL08jbDhW4eEa4GzFgVZ+gLdDcsHWMDNGeEYTAUunKGpSam9SoC0EB3xcc88ZSZgLP/JEh/UKQRrPMbE2s1wb7dljG8QWclpj2Jj1x8nxjWlXWiUHS7n2FfMCwcFIa+EHelw6HEliH1p8sNXm6XCEtO05yC5hq8MhHBck2vs8CgYZZ0A1X1IC1kwWlqC0tZoQnCfH9+TShPKsOEQFkpADhAmMwxKKodp9vQRKhUt1cAospeqBGCvkQb5ZphMUKti6MaqJhpVwr23qJY5zYagUUrgkQzU/U4m/rLIvKplSWcjFIf1jrC9dkVCxECb8eZR+QSjxop606Zqh4z7Oy5DKDXfN0UcquMzY9gKCOJsFSTTPWQs3M9ArPX9NmHy0mGz6bqCTlBnIhcOMDlR4v9YTnrrHJ5CTxvlj/PlzHzCRSkdGM3BjF1CoCu01jjt4L9+15uqtBUj4mdgWL4REd+WBKp5nTEYkUQ1aru4fmOvZrtcVXu8LLjvtWgsGLCR1U4rMoe657uvrd3S9m046ZEam5VmmqUacGYzXqcX5KKdW1H/PDkNZ6WBA37qHHDkiQ/3zyB1QE9L0gfzu+lDWXd12h3Vf5+E3eYWHg85prtYbQYFImw0wf6LJQ/gdu0oXYVzd+7aEaMbL+mKg8/6CPIN3QZHwqDgWG4c7Vs8bef/hoA8Kjc4uXAcoBjjZt+LlDrbshklXAyqQuC2mOSHZi1HssqKUpp/OmrD7F7zYSyvQaYCFJdHIAL4LO//1/j/2v8/X/2H+Ivto/xNZOYxl+eYnycvKromKnPMZq8wQZLUnkJtqzqUcg8IWj5GcqTVdrExD1ZhYUC6K88NCs9ldZjSHO8tbln0hHBROqolZcNUAZDeN5pjBygRQ1cfHorYu1t3+CKbe1qO+I196X2XYlm1kvgYq3HvKE5/ub+FH5CUjMmIPDxgs8WiAigNg6+biFoKVsoFaFsDgQOh1KkDutmkE0dYBUyvq+WqwN6ft3QliheF1kNae5z/62zBLcnxalJCxAiBEbXdyXKe33cbznZD6OO8DO5bLJ5+kWA2WXD7qgz8Ydcb3gKLG28kfuJi8LDcc+tc1xnTEGVjWKkrqqAyKpm6rqE8kps20KR65DFmuc9tc4gn3+YSH76OlK7DddR1aiOSEXBANvv8PFNlFYhRIaz06Y6PX6RHWUA7kVNRLEAMSnQI0HL0mYUIPOGqLIov+V2moTm/xbtRA/w78N4GA/jYTyMh/EbGr9F8cH//6EGDWWocd3PJhnReYc/eEYEJBcMSmbTtq7Enp13T8sWX7XneLknckxpvbRxbLoKv75fTD9KiAl6VILfeWvw+Mk9vndCZCSrPRxT5M7LFl/uLvC1Ivuxf//xj/Gj5jkAoNIOXx/OUXGGezc0eN1StrFum+SiRtGwyhG6qT1abjvZ7hq4XSGmyGQrlqEmz7ZSl2dbLKsemsPkN2qBfSIf+Zx10f/VRIWFWnTOT4mg9YeXL/BRTfDj9bjEwdN5fLG9wLLscV5R5hSiyo3+ZcCsGlCy2GgflRizf7i8R8md7D95/QF2rxbQrF3q5wGnTykj/1c/+BW0ivgHP/83AAD/+e/9z/h/Gn9j+Su8HRb4fLika7FX8BVnBE1AKKKo6ZA4OGc9lz0Ws/+Lvffo1W05z8SeSit9cccTbyJ5SZFsKsPqbggNeGL0yJOe+k/4P3jsgQEPDHjimY2e2IDdBow2PJIgyW5JLYkiKfLme+JOX1ypkgdV9dba0qzBgY7OLoDgOffsvb6Vvqp6n/cJA3R0BT9uMzbGlQuhBghV3zAojBG9kAcOvcruTbywJK3AyxlSct3Up16fOKyeb4lV3o2KmLtKWByOVUZEDCd51vLJDse+IP/cx6s9nkSz+r+7u8AxEueenm9wWrXYDNG0xEhcx/dYKRuYu/GEpLToI8rh9CTDNMKZ6Tys4eTAVc1G9L0is4bAHI/fi1LDM0bysvAuhs/a72r4WG2JnYRZOPBYBXJhoTcV3SvPJy2IFG+IgNYkaYzoGZjOHrZWZOP3HF0W/382qRDvBOSBUaA2s0D7JEL255qkVL6VIYQhytKMzcQ5KAdWOIjon+0sA4+VnjkoiCR/MwxiBBF2nMqMWGYDTCoPGfJN1bSXHmaZIHoO1krwWO2KiaduYvgmRi5zgI7VYzbCyUzeBC8zB4wxQ8QV/p6JCZDRl0T+SnBzgo+B+LkURxk9iJM8SPnAyv4nQFR6LxfVpFXjFx3K2L95vt7ipGqxUuGbI5jHwYRJZ7QSz2YBjn1xXGM/lJRmAwA6Ut12MffUJqjPMmJHuoXB4iwsIGezFj9cv0EdGzqv+hW5Id30Mxx0gX/z/C8BALdmjv/h238FIFi4fbDewMTF59vNmhibw67ME5zwwMhzjmkrsXXh3FwrwSZQLR+zjMaeGKiYvVkIi5kcKZxdCod5TDXZbGbwR5lD1hcjnArnUdUjBHcUAvDPl5/hlQ7Q6otuTdf5+2dfo7MFfrkLMpqvb09ocVmsOjRKo5bRks5KzIuwqC7kgLt4jFHLIFGJspXf+/6XOCnC83tWbtA7hf/u3/9rAMB//7//t8AH4d+eX9zhv3r+pwCA/+3tv8DffPUU4mV0E2IgJqkrAaw1jIr9LsugLmKaUDVCW5Ehd8PzxDxymLg6WsthXjaoYm93XDtKRuGFhbcM7GZiKzjZr5h4XfPLIxbVQH1IAASfHrYzsNKCp4mw0qhjSPmhDZqflJDz8fwW3x6jVpd5fHIRNjtn1RFf70+wiT35dl8SY9jy0CuVEaJuN3V+zybfAbUawIUjljY8I81325bwnlEwvDWCYGJtBIZeUe9R1ppCzNnAgTjp+sd9aJ0mFc2upL4pEBbPKQOYYMvy7yUyYaKptBmatJWHnXv4uPqIlkNFLac8AsXOw6kkp8p9U/jcA9aQkdEaP6fwlPokpQvvwjE1SBlc3HSwTuRzYkEHm47PWw6ZoFUD8JYRW9fL3OeFZ0FHjfSdBrkoeZHhWLVnsZcZ789EJwoWF/F4THlktMkwtacNpdoF1vo0dzUl0TgZPldE5yQn8gaHD3mD41RgYnse5UGw8HzCeH+Hx3u5qIrlCN5wlKXB02XYuTPmcdPPcNWFHfrVYUZ09bPVES9s6AUaG/plQyQWCOFIQgMfdu+UClJZqGgFV5SGDAe6QuEX20s6n+vDDEWchM+aI35j/Rb//uo3AAC/ePGIql1RWfzt5ilNLKK0sMlicOIRynsG0TKqiCAdWdwxz4JWL06MrkD2rNUcJl7z1W6OeTEQWWk0AjpWF97wYDMWK0TtSpSnYbFZ1D3O6hav2nC//m33e7hpQ09YcEdh4Bvd4EW7wq9ehkW1no34/e9+BQC4LA/Y6Bp/8sUn4ZjzDh9ehgXgotjjqg/PaGgVsND4+Gmo+O+GBq8jgvAn//a3UF97qO+FS5v/1g3WMWnomz97hv/ms/+SnhkfOE0YZmnBl2EivDg5YLNroFmUeJSOeo3GcQy9omfDOk6ifDHRGeqrGkIz9E+jycVcU89FCAd7Xec+ts2aPvaswzwuSpVKVn7RVKQtg18xwmImlb0X3j2Qx3RY1E+a8Gy+Pp5gJsMx/83TP8dfHj4M9+rVR9juGiILceVIkwzDwGaGUnwgcnoTAJI9OccxDjJrlwHsbiOBzwHNSUd902lknHMs6GffhOuZ/wpIWM/djz0lHLkhxN35CeHLlRll4gOnvqFTyPd0zP6yYgiLSbaJRLaCbBy88JDbbO1X3U5Ii1M9ZwwjBwIBUMeFUm5FiFKLhCk2IfP5XQHPPETcnDTNgH28P77MaTlWRNlVqq7HEOidhl5M0JKJrpnrfM22Dtrc6eYhyYhkD/C3DMNJOIYWyOQ46VFsOMW9OZX7pmJgVI2GZJ5s5ECWhAjvL3OZqORU1nZzDZIOcROTi+L9l3GedOWktH1Hx0NP9WE8jIfxMB7Gw/g1jfezUpXB6aiQBnfRMGHfVtCjRB0NyJ+ttji/DBjGTA746e0TAIFBuZ51eL4KcPBcDjDnEdqzElfdjMT3+7YkRmW7qyCiWcLV7QL7esy9SQQoEwBmxYi7sSaZiBszDd8eIm8+9XAm8DLvslNP6BV7YJVcBXgOGB8Z+MS2kGsW0nkQHU6iJOJieUBnFMbYM9RGoLsKW9PqtYQrPMazcD3VWTbGH6yE8wyLGLDYW4VH82DFYhzHebQivOrn+PzqDHU0MXi03OP787fhs7xAZxUxaKVwWEdY/u8Ol8QYfnS5xarssYzpPAdd4oufh+f0+FuH9oLj4vdfAwBefHmOTaysi08P0FGCwjcy2Nelal14zObheHfbWbDdEwnmHnDswu9Zy8P9T1Do0qCJVdvQFxjvYj5mx0N/LEJ9QloK13avmuCGlHpZHJh9GjA1O2EQ325muNYLklJ46YFYpSXGdBpCGPh4f8ZDgWI+oo0VbiUNmlip/q+vfxuvdkv6vfXqSNfmNCeHLFHakKoU2wKcZyMOANDxvdVagCGEPABBJjZO0Ni+K8L9QjDHTyYU3bZC86sC828ji3pnsfsolZKeKlM1H8EYMEZJkBc+h48XFt4p6FViXOOeTWFOosF9gwGWE22YZfcYvsXOw6rUJ/RgPvcc9cLDXcbQ76Oid8BcaLB2Ypq/VUQuZh6ADAlDANC+qiicY1raeBWerYphBMWekeynP/NwxUQGNJm9vcA9SZHljNjC5R2DOsSKvGAYF1kZoPaMbpBIJjCpFcwm5hhjhpOZC/cg2RpOgwTsPFT87hjRgZ7BpdvNQD3V8cJCLEdIkd9jbxnc+O4vSe/+FfwnDN0VsCigR0nShPW8xVnd4ib67L7ZYrQEowAAIABJREFUL3BeJRINx79++rcAgLfjAsYL6rdeFAcsY4pM6wpcVDP89U2Y2J2bSA4QtXwI/qzeM8g00XJHmsxSGHx2e47d6wX9XorQ4nWY4MrYMzveNJTGYWfZStGL8PcEQWLMiRjMMNi5IxKDbHMaihceMsLQx1FBcI/NLvZiX9RAlDP0zzTUcsD3Y09OcEcyIsUdOqPu/f28DgupVA7fbUL6zmc4x/cur7GLP/dstsEishqs5zCK43c+/AYA8P35W+i4C/ioucXOhI3Q9xZXME7gV/tAMvrVq0us/ybcg5sfA+VP7vD2LtzHpx9f4yamDg1vG6h9nEAEwqQW++C+sZQC5I4qTDip52wELQx+FKHnF/ujqtY5kWjktPFxNYOvLGk7hfCw11E2E91mTCLLrTUO8bNVaTBE/10wD75R5M4D5SDi59pBwHjkSLS+gI2OTWo5YDXv8KgJ9/9ps8WfXwVnpM2hJmi/qcZ7mmShHGQ8nhChF5jITtZy2ig6zXMAeGnRzHuczbKfcGpTOMdgRnlPW5t6whg52o8MfHRRKn8KVDfhs7pLhnGRe7TjbUkLlq8cZOxXmmGykCHCqXFTwA4iaCUBiDFAoklT6RubJTVaBFefCGUePsgkGzGGaDmyEmw8kYzq82MOWL+bBV/x5Hy1tATr8iMH73n2Gheg7yDzmXCEKIWhXmYF+ISilwFCJV2sD1aZQNCal9FSdOwVxKsSRdTcOgn0pxFern3Qi1I6T46F9MLf0/AyA8hJn/penFuJST8392iZjhrbeDnjyeQeM9BzYpaBf13DJ0vHwsPNHXz37i9JD/Dvw3gYD+NhPIyH8Wsa7/624D9h+F7AM4Hmssf3zwPk+DcvnuL16zUJzz95ek2yBQB4OQTW5EwMcJbjX64+AwC80mt81oZKaXQSd31DxgWLWU9QsNYCdWQ/lspAW07/1rYlbiMEw7nD/ttsWi6XI37yPEhqequgncCXb4JHLgwjNyQ2cmI3msYDxYTUMfCsNqgCOzFR5eEBE43gy8uWKpL9oYY+FOTN6U8zM5iLUAW/PQRKyaPFHqtIMHi9X8A4DhGv58cXrylx5zdmb9C6zHSdqwHPmgB3rlUHG7fCFdeohcb3Zlf094MOW/Kvjyck75jJAbdjQ+k8jHls/zCcx+n6COcBH6v619cr4G04hhwZVS9uacAKR9INcJ+NMRAdkCLs5SyDvEmm5R52YSHr7PiT2MuytJSP6zQP8oFIFBu2BXysOF0TSHMq3vNxV5Lxvj1W4MmZp3IQT1rEMBFIaYn0w4RD2WhKfdGjRHMWqsVl02NejKgii/pv7x5jHyFe7xnKxMb1DIpnSFap+25aKhKlAGBkEkM0qEhVe/qz5A7Xh1BWaS1yUg/z4NJBRsSlqUZyEysuWnS7iRxJZ0breGEySvCmxiRvHnbmYdNzGnko91KJ4EBkQXnkVLHppYdZGfCYQ1pVGn2b3scQWE7GBzwTcGwTfXDTbHkyYrUMyNJ2NwvPGAC2CvLIMJ4nPD9XZiFdKfsH84lf8bh28FEGxXYSxT5/l8EmSS4EY8e5wjLo5JpUm+yStlNQB0ae1a7IMLc4BvMLE//OTZbDiTHIhngkKsneU3U6rhg5KiUHpXT+1V021DcnFnKmCZWbkinltSIjf1v4e+YbzEXiY//u13nv5aL6uz/6AmpW4LO7M/zF330U/iMDmnVHjjPfWdxQ705xi0/qMMG/GE4wOIn/6at/DgB4c7UCT9R4BGcdPelzFZFRWZca3ZAjygR32Ed4z96W2I55UsdCU6+Rc4+Xh8CkHY3A3dUCLAaeM+6pnSEPfGJc7UIIcnxBE3wFAI77AB8lVu+JgzwL/UqtBfRXkak7MPDZxBBcZUvHi5M92qGg89/taoL2hHIoK42n67BYGs/xvSayc02DL45hQ1Bwi+801wTrVlyTHaMFw62ekaZ1r0u8jXpc6xmhSaOTGIwk6c3Hj24g4ow8WImbY4PjNzGU22UbO7fIDFYlHMxtBTaf2MhE+JT3LMDocfFgE5awryzkXFN/0WlB70H6OxCZlZYRDAjhsxbPh0XXxvQZ0XKyp3OFh4u6QzUfMatHut/jIW9M6lXYROyjqxSEB49Q4lnd4lG1x883gWl+6MvsAFVq2vjMyhHbLmtdU8waEHqjIUWG0d+r2DcdekVOWt5w3L1ZUu+Y8RyyoJRFWRjSGg9G0gJ+3FU4+X8V6uvw97vvK+x/J1yTlA7ubZzJY/IR6cr9ZFF3DMwypG8DY4yizeSREWzrBYKUJZ5zb3PMnGNh4UyWgE5mpyHRRZegeb6227ehHy1u1T0HT712FNEnhIPpojzIZzlQOr5eZo3mNBwAE0ciz0EuRMzdlw6ZuQc/D/fUe2QlAIIEhiR1TW732CrwB9J7BpPdlpz0MA1DVLlALxltAvQih7H7pQZGDhcX7eEjTX1ZFd8TFjeKeF1CTSVN6dUXEYpOrmxpk4v8HXpXx3u5qP75zz8Gr6vwZYyeuB8+vsVpdcSni7B4Os/wVXsKIEzkf3QTtBlzNaA1Csuomywf3+Im+vsqEXb4Y7Qms5bTojpoSZXqaCR6LaG3kSxjGE7P9/HfBHY3M3xwEhalT5dX2Jnwc//fiw+DtiuRakp3b2dnn4RzqpsR/Ys50dqBrEfjY3ix007VzQ3cLhxfbCUJ3D0DXOHw8XeD3rQUBjrqPXojg25yzOL2uoxSgULjtG4pMWeuBnzZhoX0VbukBfB5s8HgJEqe5RnJaOLvjo9hPMfdEO7rVTejCLdhlDTB3x2D7V4TmRudVlTFjlbguK/I/xQ6e+KqxZDvy4sGuBhxfhbuvzYC27iJ8QJQm6whtPOsMWVF0GSmiZ0rR3+22yLnaM5CGoeIO3fvGDWneM/heoHkyMh03jC4pUEVZQacO+yPFcWxPXqyIRLTrq3QXmXvaTUfsIzSoYvqgJ9vLu/F6FUpltAIQmJGK1BIAxPdGbpeEZkKAOb1gELm/NMkNZPKwsV76j0DF5Yydr1nJNGpCg0lLVl99lrSxtP3Av0Zw+Gj2OP+oEOTPIjbImuGlYddWvAUezY1HOFxUk4V1yFbEYLlCtOWgQSU+tvzeZ9tEBEWqaSTlC1H8zI+zxJon7qMCk2eL7O5n+iVD33DN5HMxkHP1vPgQZzeRznXWDQTUlc8D8sBrXiWuXCfSVc8nIu+jNrX2uT3xXDwTXgu5TUHt2EhBMJm7V6MIMsGD3yYELkkAAsMZ5m8lapi1ziwJKXqBVhtaRNpRgERNyd2FCEf+pD5HSQTUz5E46XHNjBKwWEuPs9/ApXqu38FD+NhPIyH8TAexj+S8V5WqmoxgDeBfZsqLOcZvrg7w09fBeautRyLWWKjMmI1boYaL67X5C4khSP2nxQO1jFKDtGHAocoml883pOLjHUMh9smGMADEM9b9LH/1LUlysWAD+d3AIDBSfz5qw/Cn7+Zh35grDL9wKGivZk+1/itj0Lv9avNCewh7+Rt42m3zkRwayHD8ZFDRgYxH0A0ef7REf/FJ78kiPBtP8d1ly0Y11VHveOZGrM0qS9hXAhrB4CvtqeYFdFgXFg8qkJF+Kt9MH34zjywgecY8PUQkIE33SIEBAwF3dc+Ga6PgpiXi0ULJSxV0Jx5ch0ylkMWFmMU5vvKUuVnDYe5DRfKHw94dLrDXXTDGrYVbd3lwCF6ls3BJ8NbBu85RKx6bC+pj8eHDHPJjQxuPRM7TOoT1oEZSb1vx2HTcxpEyGUFAM/QXBzxOx9/CSDYVX55CPfq9c3pvX7i5fqAp/Mg9/rl5gL7vqT30zqOLqIL9xi9juHQVhlWNozCxlcnRwxGYLsPz1cqiyZWWAzI1e4oQ+5tfOmUdLhcBtbxeX3At/s1dm24nmFQ4F+G41UjC73Ok2g84YP8DABwkDDr8N9nFy2Or2d0X3mfqyiq5pI5u/Jg2bsiuweVHqI25PTkPMMQpVXMBtZqESssV3gcn8XnVHlgPebc2ioywwHwSf+djwx8n6svV+Uqkw8MXjqoZbh3Slns78I7JwqXAxhscB1KcGg4UPg/L32wtkzv3CiA2AqSO47qJsKsPWBmma0b8mUjhKwZeM9QRLmN5/nn1CFWqT7/XqrO+V6AxffDiwD383gMNWQLRyeA7omFjc+TCU+h6nadYe7iVgQ0YeKw5AsPb//hd+1dG+/louqsAKyAZcD+GL7Ad7dz+FHkMG/pMKb4MuHw6i70UJwLsFZyVDocMxwplcF63pFUZpAOIurS9n5BMgs1H1EtBwxpct0X0EVq/ABPLnf4ReyDffPyFCJq1sTIMj0dAZpM8oDHz+7w+hj6h5tXS+DcZJJR6cAjzF2UBkYLcuSZHm98bLC8CBPhk+UOX+7P8DommXRdQZsFIEB4yaHoup2Rt7BzHNtjTZuVutCkkxRW4HMXoOCPF7f4tHmLb4fgcfx5e44vt2GhuN7M4S2jFB89yBzuzLLEyHkGDo8yQpObroaMbBbrGPQgydHKeUZRZJw7IOpx62bAvqswbDJZJkWueQ4M55agMhYlTQCCTSOfWFJ6FnrZAKxEvvfKhXcqSRhifFc4EYQFMbkLVR48OdEIoDwJ99d7hovFkQhfe1PiixeBHJeOu7yI7QMr8JffPgMAmEGimo3Q/h9+zZUytGEajYQZBcmKWG2pb9oNCkbLvCGcaKunjlKyNuDCQcR3etlkZ5yfvn6CsZe0EDVfSYIY9ZmBmBvIeC6my5sTLz35GHcpvSf1Hj2AIi8U0+GaiSNRlzXZvrGQ0hL0PNzWKG5E/jmZHYScAMwiYsiFA/MshM0jLsCR7GS4h7qO0D6Pm1IKl/fUR/UyHMdF3Xc3yrwha+W9sHevPHyah46Cerue+xD/ln6uF+QWBWRNqamBceWzTId52sAX29AfJm9pFwhcANB+YMMcFc9FbgWKCCk76QlOrt7y++HjIgez28qDTdoirLBAdF7yI4fcC/pcM8uLNlhoPz3Avw/jYTyMh/EwHsbDoPFeVqpFpSEqjqrQlIyyiTvhlHfYrDsiZ2ib5QF25JidHonJa0eBoglV2cmixckEFu27AjbmJEJzci6Sa4fvXVxT8s0X317QuV1e7FAKg292QcIjbhTtRrkOO9C06/QMcGeZtZqqKLXuofclRBsdVZ4dsIr+r5U0uDrMkKIR7a7A8oeBnftofiCo8KZrcHdoSHIwNdCvTzp89+QGn92FqrMbiuwRHKtKeRp2oI/LPUleBiPxu6fB0OFHzUt8O56SicbLwwrHCPdWlQ6ORXE3rUoDFSsn5xiZWhTC4qI+oDXhWZzWLV5sV3QeXDryonUTb2SrFeaPQ0VeKYPr10sy0eCGwUaWsDk1gGcQ0WSAAWSIHiQALPvILifsYc/gqUQJUC5lf3oQfOf2CswwEty7wsOusm5kuE7mD8ArAMcxBjccK/DrDMWVzw+4XITruW1rSoA5Owv/LQWOA5mNXilDOatDr+A6SUhKvegpEefQlhDSElt33XR4cxtRG82hoklEyEXlaCJCMRqBN6/CO8x6AXHkqO7CdXaPHd2vaj7S8wJie2Jiqu6vIomuZeAc5BrmeK4q+cADdJhIZMIBu1hhKcCchs9SswDhdjFRqLgR9xJfnMxGCJSmAoDvZTCXT7Plhx2hNuNQwixiVTkwiIGR8T68h4+VmC8t0ItQlSJUcEmKwwoHnsw8Rg5UIATCF55yV9U+OKO5Y5xHRpaNLDgwnsRzF+FdSoRGtRUUvs5NIDuN69hCejSgivOXGiX0roDaiPh7LGeockb5vmIAhrMst/HSZ4i9CMYbyZzEe8DHSlvdyglz2cHNLaXzAOF5+imV+h0d7+Wiumo6yJlDozS+uVnnfzAMMkYznc1a0tyVSuNkFaQ2yRD+lQ+waF1qfBDlIwCwH0vqW52sjvhkHRasL7enuN2G4y2bHpXQeHsMPUpeWNgIx95uZxi0RPsm/Gx9x6Fi4oNexD5HYrQWjtDbfVdRYo3eVEHzlVJZ1htcHcPxtm2NVdORvZyrDZ4sAnR4Wh7x9T5AsLuuCozOJBNZjdSDqwqNr3cnuL0O90AUjnSZ3nCAAe0xTIa/7C4pIP23z16gFlEz2T7Fq35F5vi3x4ZgOcY8iiJrI2eFvgfxpv6zB/Dl7hS7yG5t2zIbwVsWYLhJcs886jfnVQ4KePN6DXmdKYmu8JkpCQDSkVSBbxRUn/W9noPsAmE4fRbvOUkYzNLCNwYg+NrDRwkNQ5g07aR/xidh4LqNC/hRQF/XuI4sbXUl4RO0dzbgw9M7inTj8HRth6hJTc+NMWAZZS210ni5CYujPhRghSPId9n0uIpOVFJaLJoB8xgM3xtJjl5FqWnxFdyjlAPujjHp5q6mkAd1G97L9tMYELAYSFc7DArWCNIGs4MAFpkR7imhhQVD+3T7Z1nD6nh4Ts0qvO/dq3mevBfZ0UdvS2jLiIcgukn/MzJ3xUR6Qg5lHBhPbZZC9ZIWOt6JzDT2DKbJhvpioSHjdTrL4bzMbYEUWo6gRSddsImbySRRO3Bi8TMTFjov0kbAQ0eag+wmAeDKw1cO6jY6a+0YQbx6ZSH3gq4TX1Vw0aFM2rAgpMVSLz0t1GZpqe0iSwMzCuIChB9O7v0IbZH0vk+crszcwcfzYIaheqnuuV055eH6/PPv6niAfx/Gw3gYD+NhPIxf03gvK9W5GiEVwxdXpzARdpK1QXM24LungY0qucOHi+Bt+6jck/6x4SM+a8/JGOLNboF9NL9/NtuiEprCtX+weoO5yDmgfzoEo4lBS7w8rCjc/NHpDi8jYUdvSmyOuXLqH1noCOPamQtVFJEMGFis5o53Nfg2uphYBjuz+OA8VNCX9R7fWYSKueYjDrbEH7XfARAgwgTPJtciACikwTBIgqhkYcm83ZQch20FEeGlAPFEKOso4EsHF4/57MNr/PAkmNp/VF/jOoZBXo1zvOkWuI0aX8EdRHKHYR79qNDHEG2xzGQkKSwSpWjfl2hjZiiA4CNKO+a0I47nNRsxi9Dkrq3QJ1MI5WHONVWZzPBcfTIP1krIbcrXyn6nngdiUYIcw/OI/6Y8TApfTtm2QzYjSCQRU/vAYo3m7E8uN+SyxZjH+eMAUbS6wNvdHF0kU+kzA7UMv/PPnr3E43qPn20eAQBeb5ZZAxq9p8vohNVUA93HbV/l6kgEslSC+ru3DXnnfvLJDRaqx1fb8H7u9g21GU5XR3qHjePotES7DVUPP0gyDnAKMJcaRTQ0ESIjLLN6CMeM1bSfIXtWHyVVi6YOhvGk0TbZ6QoeAAO6FxH5GRmRe9ByIMKl6sjghM8wps4aVmbYvdxRrxw5XzHugVageBOfjc/eubby5MLkingqVU6aT9fiHIecabgqvqujgIr3Q0qXHbJ6HkqdxIotPLTMEO800o0bkIe159m3V/QMxdcCJsqX2080HU/uBfiYz5853CMrOuUxPE7O+fm/88rkir8NIQKszCQjtJlcxtocXi+7bCBhZh5iSIY04d4lDautXHB5e/e9H97PRXUz1BCihL6tMIsT1/fP3+KsbFGLRLfnxLaciwGPVJAp/Lx7gufVBj+7eQwg9IJ0E96MzVhjN1TUU/3Z9jF0ZPvdtTUuYt/roj7gZ1ePaKEulUEZv2CDK6n3AwRz7BjlGsymp+kcrQCPaSjSZPYfc4D6uMO6DHBYbxWlvHSuwOf7c8wjDLjvSgwpSadSNKmPRobA7thn48JRH+mwq4NjTpowekFQn4/s1qcfhkX8stnjr26eAgD+4uo5LmfhHrw5LHBzMycjeGt4ZsE6Ds48FvNwzufNkdJVnGe4Tr1XZWCKLJx3h4LYhSwqA2x0wVGFIYu+9s0MSBNmY4LbjUkMTk+9bzAANvfxRJtnGX1igxNOIlhKn12lDhyujJPiwoDpbJXnWTAZAIIxxPziSJuH11crcmJSzUg9z3k5hgUvHn9+ecSzmJJUCYOfbR5RwLiUlpi6fVuAC08h5d1QENvdWUFm5sVshB4kfJRnoHS4eLyl+/1X3zwnOK9aDHS8fVeiE2Ex6COMm2wWXeVoguRLHXrQ8Z1mzNN3xDqO85M9tUZ0J+HjIjUNHU+bydT7Zh5kwGArD29yALYYJv1PlnuvnrEApU7MDsZ1fBbRQSn1c/nAITb5s9zEUg8elHQjOkbwLzfAMHMkc7EVg1MTMNDzzIoVHpbaHaAQBAgP5hlBzZ7l/i7TDODsXrA6MdM1IPd5oeweedgYcs8GTtnHtnGwDe4HbCSJXh0DANI8Mvk9MEE/B+UCszm2WphmQGqNsnDs4jZ9nidHK1c5iPgejatghJH4IdV1MOyw06D5d3S8l4vqzRcn4HWF5Qc7fHoWHJQKbnHVz/GDZXAQUsziSYy2d57j/7r+EQDgab1DKQ2eLEIP66w50uK46WvM1IgfrMIxBidxN4ZKjDFPJKC37QLHbbb2K+dD3p2z+/Fai2VHk2trQ99O3KW3lFEckxOAjDvw4cLipBqomi4AvOxWdJ211Hi1jTZrIlvQWcexrJL+VuHIPBG5SqVpQgYAd1BUfUG5TBqZafzwg9ekW/2rF0/p2k6XR/zqbZCCDNsK4J6IRFU9Up+XcwvGsvtPqwvcxvSg682c9MPaCAxdJNkg9DLTeQRtoSZNqzUCw204J3EURCBjzMPflUGuBATv32xZC7nPRCIvQMHvzDJ44aEi4UYfFXisiPwHHZbxHDe3M6gtp96deTpApHQi5tFPpEqMMSJUmesau5Pw51XdQyqLxUnYkMwKTSlAV8fZPZ9eALk3zYHZrKdFVgiHoYsEnqMiNzFrOXwrSSbClcP1VXg/rnyo1JLvs/fBljIdv02SoonkKQ0Wq3V3UEBpQ6g8ADQgn9q6HHHoS5jrGJU3TCZ5lfW3Pi2OqUo7ZMtIV7jgGpScr0pPjkTyyLLNpAsLKZGQOMiijzkGfX6fbMbiRkt2AASIxMR83mCJAfRs0yKXnIekmnANbssY/xYXy8qCJTvALscLhl69DxUrwsYh/RlxcTfRW1j0nJAT0TPEsCy0Tx3swk6kYR78eeATCO5hRoEiPs+hUzkZxjLwo4CnTWTeEJgTAzVL7wCL71x8vprnNK6eo7jj1N/Va3ePeJb+OzeA2E/kQiySxSbAz7s6HnqqD+NhPIyH8TAexq9pvJeVqrrowBuPVd3ji01guw5aYdV0+O48uoQIi6+HIBl5OyxwEcO1tef4tj/B762/BgCqZgHgl90jPCvv8FgG6Ozz4ZK8bXdjRb1LJSw+enaNq33oAY2jgI5uJbPTDheLA3nkOs/wzV1kKBsW+jWx52EPEqTFtww2Vh7r9RGnTYcifvZBl5Rb2mmJapo6YgQerwL7V3FLSTGKOyyrATMVKrrrtkERQ9ad47DCg1+ErXFyqAGAf/nsC9wMMzCEasY7jiKyRW+2M+jIYGWFg6w0HRMAVa3WchSlwa4L58xiRQeEXfEuuvt4y4JhRxyuzm4wzeUR1nKSl5ijgrqLOaCFp8BvpgPUloIDvPTE0BQdA7csO/UUPhuTzz1Y6aAja5sNHNUH4T4umx5vryMScF2AWVCgO+eeepKMeyhpskRIGpxdhj7+zbEhKH7Xl6jqkcw2AEBHBGFRjjiOBTapemSenocUPQppycmo3dZkrACV7xVnHosnO3JHMpbDR+mNMRzOCuqn+31B1RYvsv+rQzBM8XexbPMsMz/LaCiQzA4mRgetLzC+aTLCcGYmchKAx/6k1zyYU3STOmDSf7PzbPjA+vsQbzpf5gCusxmErSaV8OmIuhnpOnUv6ZxME/qZiYXLx2xC7zkog5UQjhRGcFBkDCF6FsLNo5lFqvoAwHiWWeqeoXgrMPs2X5ueRSnSIw9uAXWdwhpAyIBqQ98ZSFW0IPgXDMQkBwBRWfRJQghkswod+rlyYiihzyKErLJjhDmq8Ixia0gUDngZ3rHyNvR8UzCB3PIsU6odwc5qH55Raq34gkGvHFz/7jdV38tFVd/U4G2Fb7YlFpdhsVw3Hc7rIz6owqT2neItrkyYGH+z+YbSVE7FAS/1CZqYj/ShusVbG0gvP2m+wVq0+HwMbkiDl7gswkRbn2iatL7pTvDl9pS0ebpTWJyH3u6H6w12Q16AN10mlMwvjjifH8mKz58w6LiqThfAddFhpyu8ickud4cGKoaPP1ttUXCLMUa1DVZiocK1HE3+olnPMFcjvt0E2Ng5TjDi+foArLNF3aIc8GwWNhevuhVuuoYWxKYZoNPC9qahCQeVBTwjc3bbSZoI1UzDmvx5pbK06YBDJhIB92UzpcXyJMBcwxgiypLJuBzzZArk/rNv4uKSFpiOE5ElwIWeICv/9/pq8nUBfRp1n0+PtDi+/uqMekXMhv6RjMSiarIBUcJCW0EB9R/M79DHZJ6XmyX1IJ3jOG06rIrQY37TLsgGsvcM230NNokbrBNsPirs2oqcpJhwQCTilM14T7NqHcM4CUhIny2lQ9cpcv/hy9y8n0YjMgvwVxUFe5uZz/1DI+AZIGOQQVnmmLL+qgZUhky95llOMtf5QXEfJG9xYbMTaRmPsYept+lFfp5gmVzGR4Zx7WgDxTyj5wQZwth1IgkaPulXBv1p6uFynaHg4dKSo5NsGXzHwaPNIo8EKiDY//m5gYjXaU2GTBn3JE85+Y8CZ3/bQ27DvdLrCi/+VUX3WB4YIu0j2BFGtyLdZGhVr1wg1aW0JZ2dqGztswsYAChHfAh4AC7fcldm/gYYoJOl48iD7jbZcr4qKNKNOcCVuEfeSqN+Kei5jCsP82ikTVki1bliAsG/o+MB/n0YD+NhPIyH8TB+TeO9rFT5wMAZQ/WkxWl0Gno62+JfrD+HivzvPzl8Dx9XQV5jPcNPyoDHCHiciQMdS8AT3FvA4o/bT6Hilvn71Wt8V4UQ9J2r8O+2vwUAuOsbjEZQNNZq0eGDZTD8TBpuAAAgAElEQVTQP+oSgju8jdAwZ5m9GWQLipyHzudH/Pg0sGy/Pp6QmcRgJd7u5+hiFTirR6zjdTrPsBlqPJ2Fc16onjxgd7oid6Kz6oirbo4mxtX1WmKIlXW1MGi1QhEh5LPqiM+253R8xR2RjLb7GvZ1hGuXWdQvhA9Va4ICJ2xiE7NJU4WlR5mrUQA+EqSq0x7jIOFv43a4cDhG9yB7UIBycNFIgDcm+x17kMmHR4jN8tHpRh559kVFIE+Q4YDwOdKt8GCfHHEeCUnHvkD7Ktx/0fIMJ59qzFaZLNS1JVWrNrKcx8gQd55TpWqMyI5SzYB5MWCvM1EssWcHLbFadLiIrOpWF4RktH0Bo2UOTy8skaKEcMSabtsS3uWotrIw9O6YUaKaj5AR6RhHSSQ67zKxSt4oOOXJXYjpzCKFY8BKw8Rj6mORYWjhgdJmskzhgl8sgoFAQjbYXaiG9Em8r6ULkCSi2YHkRKxhJlentvK5shYB5mfx7+qWw0Q/W9dK8IOkIHRmMoTMTahCE+TbPXZEpmKG3TOMCJmn4edCfGI6iAcGDheNSthyzHFpNyWe/FEkLd5pOMlx+E5Ama7/mSCZj4y5rkmiYrOaDGBZDsQcAwxypFvp4aOPMTPBlSk5fLFeZClYdKVKcX6icJhwjMgbGjYEjhMcrjGJcENwEUvv7j5/b03jMT6K6Mi6h9fZ9CN9pzG++3Xee7mo4nIAGoZ5NVBv8XZo8Iv2Mf7z1c8AAD+qXuC1CdDnmrf4xoTea8MG7F1NcPDO1ahYmCQ/Gx7Beo5bHeQBKdgcAH7aPsPPNkGGc9fXGI2EjBPc7ljhrQxfIiUsbo8N9RC5cKjiwjYrR7z9+QXYZYCGiuUO/8/PfxA+gHvKBN0PJbQRmMXJe1X3NNFe7eeYVwO+PYQ+7Xl9gIt91LkaiKG8GWr0RmIXGb96kFgsw8J8HAsI7kjz+PPrS7rOdd1DO47bTbgHbq+AdbSJqzWd07EtYS2n/uL69IjtJjOl3SiAIrMLcxq7RxmDuY0WwFVJi7FnGXIs1z0495hV4fOM5dj0ObTAJmZkJ8FGBjWZkH3+scAwnTgs8WfhHnz38gZn1RFvujj5vVrRTGLPR4JLl/MOZ7OWNkmqMLQ5kdKhKjTlnf5MX1L4NTtIsNMY6q0lXu0XBOuuyp7aA8tigOCOeua7vqR3J7nepL6kkA423h9nJZnjq8LgZJH86IKOt459cNEMOHYljvvc36YAduFocTTnGrIy4XkDAJ9sgmYm3szU8xT3GKHoxT1I36eJVXjgkBpykUkaGcqqMtQWccIBh6yxtk1IYgIAHhei8MGAPEwckBDYwQCghUCx4ZkZ7JBxPBYg5T7Kb+ws92/hAX2R7pWBPSrw2EdlmASMaxY2W8lmcRQkYXr8x4Cpw/G2v1fAqbwYMwPEuGDwIbOVgXBOKVXKqRxK7oQHAyPZDz8dSG/urb+nn/XcA9EBqloOqMvs7rQ/VnDpOzNwVG/iBqrL9yV8Xv6zLT3EwO7BzbQRqi19R/pdGbSucTFlESpnD4vquzl+56NvoGYFCm5wUYQd/sfVDf6g+RWKuG3b+RJ9bJz80jzGL7sgrt/qGh/XN3SsN+MS/zHqMAHgk+UtPm7CvzvP8T9+/Yfh57YLDHGnrgqD4VDSLvzsfE/VyvVuhrIwpNG0kxi1t7+4CALwuPD93ZePITZRo3mmKTmnLjTOF0cyVnh5u6R0DDuGhayOfbyjLimJ5nJxwGByLzf1OwGgng0U6Cy4w8WsRRerWiUszaG3xwZdW8DFQG1ITyYRdaVpI1HXI9q2pMppt6/h0+9UNlg3xgmVjTz0YBESflI/zh5U0IfGCZkBJKGRMoSIJ0tDa3kmzigHPySyB4v/C//kedYg2rkDn2mw+D2/ON3hN89ehnvAPP7i+hlev44kMu7zt8kzkv0kW8A0lLSU4OI8w7EvSAKjFKeem1MOIvrtSuHQjwpdRChu9jOcLo50zG5UFBivR4llXCBv3y7BlCObQu9zr9RZhvk8nEfXK1zfLVDEzVupDNq+oOOpwlDvyw4CPJ6X6yQVo6wT8HcSnGLWHNR6oHvvDQc75N1Kkm2wkedFFwjPKC1sJkuk+KMRp6sjJUftdzWqX6ScQkDPPcwyr5Ziomf1E/OE5PULAON5tjCUO4HhxFHVKdtcadvCY1y7HN/ncK+6S7pmfqVQtIyqx/HUUXKRFwzyKNC8jCjLnSLjg90nDMNp0qX60ANNxxceIlWENnv3AuGa06JtF9lGke9l6I0m8lcvyXhDKQtrOYbXYW5QBw71/fwubW7n8EmnnXSrCOhLWtC5Cak0VJ3asOADWf+aqubxic595KMiZIp3PBt0ICzurvD5PXiHx7u/LXgYD+NhPIyH8TD+kYz3slL9veXXqOYSK9FiIUJFeGWW+Ov+A7zSofLYmhq/2Ifq9KgLsvl7Um4xOInP29BD/Gp3QlXCp2dX+LC+xUaHXeD/+c0PcYwuPpx76lk5ywHNUJ+HzzaWk7GClBb1hAG37yS6fdqRx51n7FHKkdHuvFn09Hu3uwZ6U5Ho3bPM4pMLjUIaMqH3npE7D2ce2z4HSTsryBAcCJZyQDAfWBcdsT8FLyhQvGsLuEFQv0sWlqruQlqqpj2A+ayHTTKOiTQGANxVBTXJFrWxGtVtkfsu0oUUkCiO907CRcP+47YCE54yWcdDkZ2ouEcdzfWNEXBfzqhXOpw6+GQcvtA4P9njo9jv/qi5xas+wLM/v32Eq5drgq8892DzLGlKFfmru8Dincd7J4WlezXqwFBOFn1KWZSxWtRcUC5vIS1KZXDsM5XybUyKSfcyjZNli+sX0YLLsmibN3nvEty+bLGPUhszSKjKkDn+bt/QzwVTDgGX7B4nKSJs5CivYj9YephFhl3VOptcOMfhBp6J04WHPOTf8wy5/8pBkLI4cNjH4b796PkrfHV3gsPn4dqq279nMNBYiPjc7F5RTxsM8MkycuQYZx5yHm0bS022imYZzEPusYaTMYQIkG/KWobm9N0Sm0kFZ0Mf1pLVoaccY9cLyINEdDBFf8bQPomogcpSE2FCeEBiUYsBhKI4GarUlGuq1y6HP7DJfYvm+hQIYBhMaq0cGJwCeISXzcxBb2q6Lqjcc8bEolP0DEXwuwmWkSr3dslxCqF/a2Y+V/Waw8Y0Lt7mcABbAGZu7xlMMMvIPvFdHu/lovp6XKIcFf6s+5hg1+1Y4+P5LX6yCIQkPdFf/P7Z1zAu6TctnhV3+NUxxLVV0kBFOPZuaPCX+jmuImGoH1VwDgLACkuWfPquBDMM7U140bvS0mTVVCMkd9jHRc8YQX1ClA5s4HBxkjj7/gY2voTbfYP2OvLrlQ8QZ3y6sskLY90MWNc9QcoA6HxHI8i9yTsGqbL0gTHgk3WQG/3G4g20F/hqH/xgBy3RRd9Y10lAOsjYD02JJunn0gZECIeqHGmBUaWBi7Ci3pYodlkfyhgoIspJAKt8TD9y1JdpgeRh0U3/1nGMsZdUzEdauJ3hlAJUXguokaG/CM9w9uGOYtRqqaGtoM3Di26Nr+M1X71Yg405jcY/HglONY4TVOsdg3WMetPOcohI+nGWo5oP6GM/sO8KCsKGZcBJ7KnKeN7xmekxh4YXpcE42ZBcXy1IkoLSQTWatLpcWOqVHrqS9KJCOZyvDqSLvTjd0ftx6Ep4xyHjhtA7DnsXzrd6JbLOUwXSDIsEsLLS1Dt2hoG3IhO+SgsTF0DWiTCZpw3PZGGzTwbMFuGe/vUvn0NdK2oz9BcOWEdIszJwlsPE9J+0mACRyBO/P/NHBzDmsX8VfZ+3NXg52SSY7FAmjxlmHS8s+EwDKYZuZLTg8gkhy9QervJwKYJu4GCvorTsNvxQ+3gie4kfLQ88Wx1ahmKb4VQAsHHN8zzYKup17JEfRHaEskHKAoDCyUkCNDDq5Y6nDmxktPnxpSO/bGYY1JUiWJZZ0OLOTdbB2irKe2Ib3hWASSk48dzSM5CH7MNsak96bfjQZ07SJLUL9ot2ePfB03f/Ch7Gw3gYD+NhPIx/JOO9rFT/9OojyLbEWd3iB4vg03tZ7PCD8hX6aAVyZ2b4z06+BAC0tiBDd8Us/pcXv08kIG0FFtHpxmoFQEFGRnG7W1A+ptM8e5/WNpByInTTLAYyuL/ZzLEZ5+Cx0uPMo1qGfztdHFEISyYPxnF8cRVcn9xNSTtCN7dgZXa7SbAzEIT+m65Cm6j9zJOLjFSW2LirRYeb6wWRh/hqII9jAPijN9/JYdUmVxessJCFJRKWtRwmmpwK7nAyD9tb5xlGI+iY3jOSXMDdN2pwpafqXMxMZjIaBrUYKUfWDJI+13ugOe/ILWo/lNjswjNzvUT1NlZRygdz7wjdMubxZhcqmfP5Eb95+gImnsy37Rqv4zUzzeELB34W83fXB5iIevQ6V5KqNoEFGiu/0TGCdefzLjB/U5BALyhVBwtN1+Icx6LusYnQ+RSSDwxq5GpUOrjYLWDR51VGCNw5TmQzZzmxf0/X4b1KEqnRCrTkw+xRVjqbdOwVqgT5lp5IOXptoVYDnXPXlrARjmS9CKYAEwJSamV46UMiTkJjltnwwWuO49fhfsuewZY+hHYDkKd9rtx7GSRR8fDBFSsZK2RE97ivIL8tIWLVZitPhCPXuGD8Ed2EvLgvh+EvK8TAqcz8BQAODKfR9Ul58J6j/jYzltNwJaDnjo7JR0YEJLAIHQNQu+jvGxF8bkAXoOcBrpXb+H2amM875XNgeeHALINN8qZJZSoOIvgCJ/RqIqmRe45izyb5sJltbGvApC6UDuec4GtzZrIL2TGdG+gc+0excl+P9H77MRAH1S5e95FBdAC7z+t7J8d7uaj+4eXnKOcKl8UOQ8QfFrzH//z2D7CLWsCCG3y6CJKYvalwiG5D/+HlB6SfBMKC1U+caPpRoY/uP6Jw5F7ChMdHT0JfVnKHUhiClIGQNgJEKFUqqAiFPl7uISIVMGkQkzxGMB/6ZIhMv6jJrGYjVrMO501g9XHmyZB+35cYddYaWiOzFMRyctm5vZvBWwaxCN+qqh4JBv13X/8Im9t5nhgNz33OwkFITYxTILOVBffYxjQVrQU49yTxMKMkKYXc5aQZAPBrjToGaDPmkbLfOPc47qsclsw9QVlirnEy6+he7duK4F++F2Qtx1y0N4z3Y381h4rOP84z/Pn1B6QLPnYFWQCWH7cYtKQ+5HEoaJJXIgR7A2GBso5jiHPYrBmyZnVUQWdrJ3Bl6ttZRvfwdNZitIKeGeeedKPeMwiRWb3jICHLBC8zcOEhIrQ6q3tKTeqNIGheCgvrOO7is/n7o+8K2Li5kltJ7GgnARf7mihtsMKLz5APnDSffGBB15tg6Wn6TGWBkYd+JgC2FXnR4nlhM0sbNKzx/uheZjceG3qciQWOIm+8YDixc+WVBPOArR0dnyICNb8Xq2ZqYIxSELELMCsZ9s/y5tIVeaEXBw55nBj713mhcwqA8FDbdH8y1Jyi64AAOU910k5mlq2X4R6mIAGvfL7/i9wSYcC9TYboOXigTWR5S5sSiSwQN2TyyOAEwHM7GjGpMThkxY3umNysEuv+KFDcZbmNaTxBxa7IGxx3UOQ+JXqOYpPvQZLUTAMZ3tXxAP8+jIfxMB7Gw3gYv6bxXlaqf7N7CmULvNj+iFiah67EosmG5UpYvI3euW83c6ocOfeYNUM2RWeexNLHroQ1AidngehiHMcxYnGnJ0f6OQ6PTV+T0QIA3LSBOGM9Q1ONBBe+3CzpHD2AUlo8mgdIc9PXZGKAyhHMt44QazIIqKUm4lMfw5DT9VjLg+8ugjdr0tL6bQEGwMVd/v4g8R+uwzmKxkAUlionSEdVqywNvMe9qmqYaEWHJNJnwVEpucrAA2KbIVmUgD8NVeFs0dO9Go2g+7Hf1cHcIFZmzWwg4wPGPG72M0IVbC8yIQO4FzbuRYYBxULjbBUq/FIaDEZif4jB28KST+9xLHDSdNjHKtY5ThC+5I5MRTjzGGzGstMxgVDBc+EyZG04bXPrxYDT+BxHK7DvSkIlLlY7Qjm6UaEfFP2bKgx0JAh5z1A303uXDR+aZiAN7X7SCgACOS79eRwU7MhRfZNIQJMgb5EdfVgro7g/3lfpwWPVapvoRGUTPDktxdi96sTO87s0NW0P1lfsXoYnZX3KENCdPek9vUvwuBco7iqXSwnpqLIWB05ez3TOiZHbMdgK93SaFBSuGVis9IodAxzIpcksciSi3AnUb3g281fZiN/L7DzEfCACpX9LulkgMGaDxjd6Yj8/Yj0LzPrRSGKHd9sqOD0dcs1EBCEO8JmmucgfJcqrlAQQrjH5CZvGk8+wrXzOU5UO7ChJC8wHQEeoWa9wzzjDT/5cvhUE93IdrjORwcQQru2fgk71vVxUr9sZBEp8cnKDl4fQvLhYHrDtKhwjS3O5aEmiUlV66qMOKRxNOt2oaKIaW4WiyTCM1hK/9VFgEz9vNngd5Ri7scJp3dLEeNs1cD7/TjtKlNEJ6GTWodN5kpTCknvO49kO7UlYBLfbBh/FhJPDWODQl7SYsQme5B0D454WTzeKbDhgeU5dmUgLgGBv5n0WkzPuSQriO0GCfecZnBEEcXLuCZrUg4SM8g47ipzBiLAY43n44pubCn5mMIuQ7zBI8AhVai3QRgkALyxEbci04PBqThOoLxzczNB5oKI5HcwKguXsPBj702I2Mbx3ngXYNc7Wj88OkHE1Pq1b3PU1Qf/es3v3OUHl2nE0SkMW4Vqu24ZgYuM4nOVZbM8BtY6mEU1P88ugg/uWjJDyYCS9L6OW8I4Ro9hoSRPmen1EU2gKMB8HSRaJs3Kk/661QFEYeidSnxaIZg8bSf3Q4dRRILUvbbaZLEKKSYL3+JidjJiJVoKpn9gzWrBsE/rlyU6SiRA4DgRIlkVHn7R4p0Um9fUAQO2DE5KM95EZQcx3W3vY5E6UFtgmm/eT7ESGzTHBqT73LJ0CxAjqgToFuDot/IBK4fU+yF3MafIRZChi/7nYMpLEABHWTtfUZRjU1OE4yekJLLN57TxsWtiEzZ94DWWZF0qmHCA85h+EDeDmZg52TJqX4HIm9xnynU5u4yqHinsBjGfxIc4MGfSLbbAoTAu/PsmZqeIoyKoQiPaO8XTFMJHhOEDtM7vYZUOsd368l4vqsuwhK4+XhxVZxPVdAWcYVXvLasB5HSrOSjY0SRbcwniO60Oo2owR5IizOGmxqAbqW/348Ss8rUMzo7MF2tiXPa+OOOiSgrwX5YA30cauPxSQpSW/3x+evMEQRWCfbc9x6EvqUXI2xybaAa5WLV5uwhds6At4fz8RJfXnyip8+Y7b/BZTH094kjZ45cEaky3jJv0nAMGrMx2CIcsgBgFZ2nuVfaqUlLTkzQvmoUoLGX9OCIfjrqLjFY2m+wrP0CXrPS2oMmXcwxwU3NtkoZc1iXySBgKEzQPfS7o2SiAZohfqyRjvncIyEs9ujw2OXYGqCf/WaUkbIckdRiOop8qYJ9KV5I6QhkZpKG7pWU/77+OgQoxWvMVq3WO9CJWH4A6LuBD3WuJ4qPHBRdDLvtosaQHn3IEpj3HIx33+KPzcsuzxcrek+8iFow3g7W5GmxouHIY+bw7hw6Yn/dmtDPTpRHqS3pfbiXyJ540JEGQi1K+0MQ4s9dZKDzOPG7LC3+uFe+vBitQrzbIw3oZeK7kjubxoyzYsltEdFF7mato2ecJnjgWHo1TS2hxAz2xELZJDYp+dkcSI+NmgnyXJC8sELK8As7L0XVAbQQQkp8Iidf8e5eo0kYBsHRyUkgRGrx3kWXgnpOMwR0WuYX1b5IW0Ar57EbzKnWd4tVti8zogbfwoqEJmPvZYk7JFeuphexnvUbwAu5g4VF0V9DsA0D82OVnIcIjb8P4V26DVnS6k6bNNFSRB6TymHs1AqPQnf31nx0NP9WE8jIfxMB7Gw/g1jfeyUu20goxpLwkeXC1DzzOJ9t9sF1Sd3hwa2hF+fHaLmRig67D9unUNVQ19r9AeKzTR9/Vnbx/hqzIY8U/lI2/EAse+oKDg163Mpu3cYz7rKZB6rVrs4ja2URq9liTdeHm7pH5c2xfQyfx6p+ClR+oQT92cpLQ43DQZ1i0t/duwLwniYR7woyKzba4MGTpYy+GVC0YPAFiVK07vGaSydL+mTZIADceKpA0sUhWlSm2bvZD5yQAzCkpAuccI5J6cesyuADMMbpkphDw5OSmLMZp2A8EtJ4nhvfTwCd8UHs0HO3xyGqDzs/KIl8fQEmiHAlWlsYi9x9FItLHSTAkv6TBSWjK8Z8zf260OVlJQ+JQ5zpi/l74Dn72KC2Gpwl+UIxbliF10uyqVwXbb0O31lkNEWP3iZE/93BfbFfpRUdqNNRJD6jVadr93WVpU81itd4o8X1ll8Q/6XNdlut1wMSyBcQ/fSahNcuzwBHV6HryAycxjZLmCE4AXjGDvsjTh/USQf6QqyhU+wIexujYzT4xt0yB43RJz1+cKKHo7AzFNxXEgvS7DxPM5Mk9T9cgto2rUy+D/m87ZC5BpATioErYrC9Zx1K8SNwBUWZt5vhdppGN4kY/BNDCuHcST0E9XLLSV6DlxD7uNntiegUeDkD94+hVkLCX/7NVH2LxdgEfI16XMYADFlYDociZrgKzDH8elC8hBYg0fsj9vcswCgOKiReEZxk14wPXXCiL1m+M9IyOKGugfRWetuYG4U/RzXuQqlms89FTf5fH6agV+qFA2GmfRmLxWGq93CwxxoStnI17fhbdN70qCRT93Z5jVw2TRAEFvtg1G1vsxWoJxT/0z7/OE6rcFeMeyDZ+KBA2ERIlFNZAU5P/4/McEBdeFRiEtQYh2Eh+nxyzNYDxMPmnR88qRfMI5DjUf78Gzh7t4vq2gFBDPQm+GEUToJ8HVGn1bkG2bNxwumXcfJQxTtFiodQ+deqqjJPmLOulxumwpZN2OgnppKC38Lh9fLjQZ7xeFIZjS1jHoOfX1HMu9zLsCKByRWeQxyyXAGPiTAKn917/9f+PH5Qv88fFTAMBn7QUFBXSDwuXqQIvUrqvQpWi5VoIVFvNVOM6jxQHPY1D7z+8ucRHlTKMV0FYRocTWnM7x0JUwltECpltFPd3niw36eB6CO3Dmsa7DMb69XdPkw6WHqEY8OQkecs4zSsRJC396B+1BZrelyWC1gSwN+n1aBYH6NHzW0Ct4wyHj+2ONgF9n3kAimrm7AuVNJmSZxgepEsIiyhwj0s40fs1XFkw66Nto0/lSYhmTFb0AugtPz89Fog4QerbTWDFbIpOCJmb4Tk5kLWUkUE3eCQrRZuH3klOSFxlCdirAmS7Bok0+ZrpfACA2EuXtRC4kPR0/haon8pboM1wuBsCp8Hv9IwPUFi46gzGZ5V6sF2Ajo/tanLb43WfJAY7jp9chBWvzYhnkSCtD51Xcxc8dI9ks7smGM5dhXuEBw4ngxHWW7+B8QBXfAaMF/NcNmk2cGwyytpWFhTT93nBpKQi+/qrIUHDtofb5OynGCOPnjtU7Ox7g34fxMB7Gw3gYD+PXNN7LSpVLB64cvAduD2HLZgyHuamDLADA8LYBUmAxw//P3ns0W5Zc52Jfmm2PvbZsV1sABBqgeeILiUHqhfgkRWimqQZPEdKP0N+R/oFCE4VCDCpeSCIfPUEQBIkGGl3dVV3m2uPPNuk0WLlXnqs3xYDFujnoqOpb95xtM3N96zMQkchS5Ba91chjlqhzkglBLUjScBjizNWpR8omVAHuxCJEKOSQOFMUBqum5JDoorDIdWII1JnBXkVziYP/LwQAk1iId7IJM59CrXsNrT3njO67HFkVIbxRz4QmIQL2FyOGzgb4CgCczRCcYPG932TwkQQ0GFCw+ThShe6NZNPz2bhFZzRnqMrMQwxC/2UOyEDQIwAhPfqIIPS3ZcreFAHoUnWkpj3cKgWRy51iblX/2LBs5uh8gz988ksAwMrW+BP7XVz2hEr8fHmO6xWdbFkYNCbDZfz7qOrQiETOyUrLEO3NrsayITLSuOgw1oSDvemnyJTDtCB4s7OajTj6Xt/Js6xmLVexP337iCPixnmPTDp8uyJYul0V7OesMwelPC5WdPzeizvhBL7V5KcLkOxq0vO/0weyKntVMRojJ4bRAN8pCiaI/zbLHKEU8bP9OjJCWwkz9alCLHwKdMhiPN+BGT8zvyNsPLBwzTgQ6QkE+dq543+HzPP9zhaK4WXqVaSqx8w8s2eDxB12q+hTCH3QyVxCNQK2Dgeh3+EObBw0YKfx3XCCjeClTcxk3dAx2PFQ4abzVY1AcasTYeigurN1QH928C7vNBAdvkIvkzzIC7gjg/kZlfLHoz3nIi/2FbZXkalVeMAKlkEJA4ZngyAnJHs6mAaDjfjllljZXE0+6rmd4o2E+VUkPvVA1gjOVZU90B84QJlJYhDX32iY6AvcPDMHzkvkd5xt+LSh+gD07z7++14uqt5JwMo7RuShUaS/irT8IANkhM187hmW08rBOIUPZ8SwxAx4viCrwHyyh1aeTeLbNoOM0GE1MqhOYoJKl2N7MaZeB0AuRLEnNqk6kulE7RsKyxDp4/EKrcvYas4sS5IyAAS3HkB7hyHQHzy65UX10o1R5oY1bX2XMbRal8k1aX05ht4o1qYdSoXsNoOsUyC18CKZ3HsBXVqGpUMQMAOd3wmMjqhX5LzA8maMMroXhQB069irswKh9JBxo2E2qTcKAb5HaicBAU4yYRcd0CRv5xZHDwkWVTLgJEKyp+UOb9oZ/9ulqfHzJQWtL3cV6rjh2GwrhFGL0yn93qYt+DtUbcnRKaYQKeUxixDvB+MFWkfX5ruzS3x/9AZf7inx6C8vnyVzfUvHn8/o+KUMDC9PJ3vODr21FYxT2NzSpKnrdH2tUbBGobEvMeoAACAASURBVI5ym1w7bGVktK8LwCTYlcKp6c+zacP6W29ibml8lmazferZOgFROA5WCAe6UtFIqGaA4n1i8sbf48VsQpK0YYIGSJYFAOgI0hwWGN0Ktrg7hHtFo5C/ydJCJQlSBShBRu1SWoywgidyTqsBAE+Lm2e5jePj90WA1yTFohNNzl5BRc0pOzgJTtnJtoklbEe0mAzHJXtxoG2lBZeN+F3qqfYnLulxe0kL6iBfaQTDs6JyOD3espzq+bdnxMIHiJMxZJ9uFIprmdjKHpwiY6aUOyu3w2ZIsEwpCKB7bJg5H1oFHyV2xY3isHSfJzN9gBbp6O4KvSNGtYvPnK0DbJxDRCt5M1IsBHSKcQUEYGoBp/7j9sS7Nt7LRTXsNIKP5KDhpTWRqcDU86Sl8znFRAFAZxWafYEvQdFvbZshxMV3PGoRgsBZTDlZqiqliWjH8p2h1zVMQNW05arkfLTFGz/F8XzH3zeI9H2QuNiO0V/F6q4X3MvUI8NGEGFHFmWjh8nEYIh008pDycAyC6UdR7pJASyiP67cKtiJg5zGXpqTcPEFgwrwjWZrOT833FvLcgulPPZRHiNuM0TrX9QfpG3p4moCoZJNobNJKuMBiowaQq0FmLAiTLovwlP1MMS7HU32uLwiWZE8bvFktmVymAsC44zOc28z9vN92Rxh3Zc8/zsn0QwGCaMWZ+Md97cHDfNw3bwXmMVYu/PRFg9KOj8tHR4Um3hNA36xf4gvVrRor7YV+gV9jmwlwtiy93K/SVWwGyUzCQDY7EqWsijtuI9fVj0mVYtxHpEHk6HdxhSWvQImNumJnYSNxhCL9hB6EJBTg3qUrCCrGGrdSkJZWIMrgXAdkZK9TMHgKkAe+DIDEjIm1kjpoVRAP6TWdAqIiwH3Vgc1j0jRZrIXKKJH82C4MPTqfJ1CucVWQ5oDowQR4MYHEqCIeIS9ooVtOJUDJMNVngwTVokcx3aA1cGiB0Bf5qxhNeNAiTMAQkkVYrZMGtBhg4AQK+P4SB8GjGfHLc7mNGc4L7HaVWhjRRemFh89JBLdLKc54if/9CFd8o2CO6L3U671HZJV89RxhKHcaK7cfRZIozpcnpB0sHZuiUfxluapvBUob+K7Zsl7GBisCMEbFzvySZ88GGYMJC8J6NhTDTIRt/oZYCvBlapu6ZofSmze1XHfU70f9+N+3I/7cT9+TeO9rFTZuNmKu44qMvDOOVsmSzHowAknsV3JqR1mVSCL8N35eIu3mwluBmodgCJWCbsu5z6V6TREZZHHXmZV9JiXVPHUusdxtcfLBfVKpAzII1X+5XqGxYsj3hX6uUEWYWMhA28+XeExOmrwaLbm4xjFSuaT+Q1yadHO6Phbl+HNhqq7fZfBX1EVFUYOamLYwhBeJKjZA7CSpSwq90mW02QIe80m2vJhi9/7+CsAwG03ws++ekKfIQOyKvXuhCDjCABAL8lNZwhnLj30eoDiA28F7dhDHncYxyq/ygxm8yhF0I6vKQCUKsHXUiSEYtHVeLueMOwqlUMepUNKBOz6HNdL2qK7XjKEqZTHRye3+MOzL+i4gsTLluRTvzl+iTxax/zl5hO8bSYpBEEd9BZPOkiQMQUACB24v90bjfUQ4G4VzDbnSt50GkX8d6fjHTqn8GYV7+G6TNm8T7aocoMmssWbfQ7fDjighIxh3SInCHi3ibKf0nCSjjcSWWUY/vVXJUOa9thytSizKL1hMwLHfdMACYdkxwgZGBpWW3JDGqpM4cGGDNKSlR0QK7uRT/Bs5lHEahqTDl1dIruma2zrwFUaZAAGCYoMCCrw5+u9YBMKYclMYnBU8mWSocALiI0mQwsAEEDzJL53leVnwkWnoqEq1OnxIxcmDbTn8bhmhq9BsAqLbZozHs7XwJze3VneQsby7fV2hovXc37/ASC7jhKVPuWdmgct4AW3dXwl4KIMTzhB1/HA/ISNGjYKwmmWxWS7BMubyYGt4pCdO9yKpUxOZjpQ5TuABsn8Ce7IcotKXmnoBmwooRqSQcn7nuq7PUQvcaCMYesuIPVF6B8iQaudIseXQX6QBZxH6EZJjw+PFmwj+HY5wW4boT4Z+CEvayIEDVINJQPLJ365G2G5GjEJ6WS2w+WWJvXVskYoHWQkb4yqnjWIIQj2fC2nHR7N1hhFDeiz0S1OMoKCf7U/xd7mDItaL7lHs78a8YSMLJAd4bCI6CTLUdLDB8E+uwMUCoAD1c8/o0Se33/wFV42FOz9j18/Tmk2IsAVko/DrPLUg5PU2xp6gcLINPk5ARclHcW0Q1UYTAo6z1w6PJmRg5UWHrXu2cWq9xrzfIidk/inS+px7m9rQAboatDxeobsfRBY7aoEaZqkB/39D56jUj0W0cbn6/0Jfm9OmwcTFP5x/xgA8GJ3BOsluyNhDN5I2F5BSPBCmuWWHZq2uxL9YNEXBPJJfwcetnHR28R0nCH+rpq03KrQymHTlHeIcyJOpnrc8ubBe4EeGds9ei+wu634Gnsn4WMwufCAjyQXAbAMKniB0CqI4Vo5zWuSjZ7ActgACurHAmRT57O0kB7qFPtpgDk6WBwPHL9IrhYh2CYDVIB5EI9rrw6eM7BMTGQeodHIL+k9t3UiIwGBouzi9VE7yVAtQL3T4e92bnmDKa9y3kgo0EI9LKZBkO0fQOdoj9ImJMsdzo4I+7xZj9DE+aQc92itRhafwatmhMtoReiiz7WP5+YrjzAfYiAFQmyXyJscIQ8I3KeWydP4rIcQgFxGudZe3JHDyD5JfYIAOtrbwxfJeUm2QL5M18bWCbJ3Q0zjgIE6kTaRRqL6NrV0+mlAsYjXzpC38WHk47s67uHf+3E/7sf9uB/349c03stKVXQSQlJ+4rBLk5acV4amvbBpxyYOQreRe8jcwccm/Xy+57zNAPJpHViVttNcufkDuE1rh+1tzX6ncmzQjQnC3G5L5IXF5w/fAACO8gY/vXlEv5c7yMqg4krEw8aqp+0zjCb0GR8f3+Ks2OJZRQSHGzPCT1YEu9qgkMuEyZTKYL2J5g8ueqMCUIW74x1c5QZdhAB3+wJur1HE8PSqMGxQsS9yTMoOv3f+HADw94sn+PL1GV2fVjFLRE/Ig9gsY/UlkaoGB4SJveM2NMBLfmJZUlBk9o6J/dbkcBHOX+0qCmCPFdzj4zV0vNmX+wlVqADgATlAhSBEQcXP7IymvNMIgatpj9999oJ+LQj8Yn2Of3X0EgDwPzz4E/xZNJD4q8WH2PaRFSw9MungIhPndpM8d/OSEn2G+9kZzWxi7yWjBn6n4a5yiMHofGz53igZ0FvJ+a2jvMeup2u6aQglGa6BkIG/2xkFxEo10w5aJ4ZvtypZfmSNhttkSZVyfKDOF4CKciA/GHcMBWfhmdwjc4KCB5hbLTS7W7kSQAC6R5Fws0+lii895/kGT6YmnOtaGSYIopeADizXQuHIiAUg9nKUjGCnoSxByUBk4A4SoDzQOQ/EpYljNm2+IIKTexSJXCE+yyBv4YFBrBoK2nbxke5nAe5xJAFqj3HV42REaMlptcVPXxOa0d+WzIptW4VFAN8L1+pUdWsP7AndoIsCiIvYtnACLqoJ/ITM74d32R4b9r1Gp4CDgPTiViSJkQL2Dz1wHhn5NwXfJ9WB311hqWof2Ms+PyCaFSRvErEdhGXOftzZInkQuypA7wTKm/iMKzKAwL8A84f3clGVRkAqAdkDeXRl8Zp6BmyOrcCaVQFgOo09z6JHqS0bpu9NxmkwV4sJvJcMyQrtUUVdoBAJ9lMqLswHEph97OkNOtcXa4JMV2XFMpdR3XHsGQDs2tSnnY0bnI9osflkfI1CWlz0BBv9an2KraE3fZInDSUAfHF7DjtoO3OP8SnBxLm2sE6hj+d2uxhBDS5MIkCODOaR+TorWo6ZQ9Hj8XiFLzYErz6/OkZYRL2cAsdW2VYTtfTAnhFxUgi9ZLtEgH5HDFaEJiXgLLcVnFUMlWvt+do/PV7CB4FvLqnPebkecwTbrssZiswKut4DFJpFyRRA/WHfKeRHtFn5/sMLhtQr1ePfnH6Jf1V/DQD4SfsMXzckrTJO3Qmgv93VWF4SI7ect9x/bpscvlfoY39e5xZ20DV3ivWl0gpip0bt4mTacOsgUw6jcc9993VfMJwvpUffaxSRHd0LwMSNSjXqGSpfdSX2fZZY6QDfF7fOSOIU3wWlQ2K0F2lz1uxy4iUMAQmlTcxrKxGMhFpG9m8ZeAHQG0Xv4208b5WM3McPtry49G1c3IdEJS8TLB8E4O5KqngEJEciTwzhAcbtT1xilfcCIQ8Q8TxDJzl4uz9xkGODsI2riEzyIwQxyNkRpIwM36g5/3jDNpazuoFxijc8i/0ZiriZqh/3/HzAx03NMGSSKamVvpNuI1vBTmyitsA6Hp+RCCMLFc/FbTVCnNf0WkHvafGnDwH6uMnoHlhAAvJyiAIE/56pA0K8n8WVhsiSFaErA/fZsZGwZ4Y3Hcg9xMBbEQexgZIclWwZr38I6CoBdxBg/66Oe/j3ftyP+3E/7sf9+DWN97JSHXZ/2U4y3ONzgiQGX81D9xcIMKTb9hmcE2zOIGWAHRiyvSTyUyRGqCpBakAKBteStJWDQHo8ahk+LZTDm8UU2wgDrnclf5fSDkVuYeLfhQjs+fqbx6/gIwZTSIta9rh0tPvtvWIGaJ0ZvNlP8eqaGAj2ugRGdFzlpOPqubcafa/5u2ezPfsd91ahLgyqjI551ZX850ejNfY2x4slfb5pMsjoRiVV4KpYOIEwssxelsqji+QY0QuCkIZiQAp4k6LNbi+pAs/HPcajNgVqW40Pj8mUY543+PHrpww56jr5NWvpMY5QubEKWnt2yBIioG8SIUhVLmXn2gyfj18BAL5TvIUJGj9tnwIA/vji+7jajfg4hmrOGI1ulyMbJ1zLHJjxC+05IMB7SUHlIJ0wuxNVHmqafr9tM/hc8u/0ykPHytUdVMj7fYEsc4xmlIXB2YzQjAAkAtyqpjbFAJPKRCRCRGyGqgcAX0fnEiojVUAxa6EjamCMZv2t0p7cl6J7UdABcp+IRMIl9yGfgZnj2+sRH4fsBTD2DMWHG4kQq11ROgQjuToSLkGahzFkQURP4gH18CLJNVWA8AL6zfB8pmBvOTLw2wwqEhndLLUm5FYzuclPLfJph//8A2p9PCjWuIpOXX/++kPOZwYI8bpdx+fltkx6eUvP/oDowAnoqJ21xxayshw04SpwTqq4zRmyhxJ0faNjW+ZSFqxqCaodBAr9iWOYWO4V8qVEPxsM8N2d0IXqxYD3EnRuDtyohvtnTyx0ZQmJAgALRlxUd3BfvIArwN7IQQp0RwG+vWf/vpNjgH/7aeBewBBgzPZjnYAdRN2BzAMAwEkNYSQ/iP6Q3ZZH+HFgYtZ7npC7LksyHKXgncTvPKN+XKkMsriC/OWbZwheoI2OR6FT7LjzwckSEuFODusnE2LZ+iDxto1WdWEKKQI2PcFIvVN8HNs+x3pfwkQ2J0YOqhjs6gRsXLyy3CJ4wVClkgG5jixVrTArk6XepOgwyWiRal2Gi+2YNxPFqGenpJB76GgIoDOCvwf7QXFgaxiqKGfoDoCUoWU3sqjHyYXJesmTvJQeq8i8frOe4sOTW+gz+tx1V2Ibe9+7pmDorcgsMu3g4mai6XLu1QUvEAJQRwh/VjQooyXO/7P5DTzfneD5kiDffZcxc3doEQDAjauRVeZgsVFsD2htQAgqLVKNgogLgx8dGA6E1I8EgGzkMIkpRrsuR9fmqCIDersvYa6jtmJikNcdp+w8Hq+w6uhnq67k+4dNBr2RfI1dGXhSF3WCcQHqncpBZeUFL5y+V3BGsgkIkGw0rVUQuQPYCStBfK4kJ6bELBUI0VUnWJFSak4N9UeHnutZBzn0b52IMC/9XZjkEuSKwDKZENnDMkL9UgXYofdq5B3Gr594qPisum0G0Un42GuEldBRyqIbgfYxPROTkx3+4MlznOXE6v3jN99DH6VUubZY70p261ptR+SIBgCFh4xWhNKQQ5OPJzd6ukE4j+eyLaCURxkZv6bXMPHW6AsKDgeojeWqwCzkbHPgUqUjI3mQNO0kxCBrc9Q7La9ifxdg1ze9VsmOsSPb0mHu7OeeGL8AZGXJZjQO2Us2hLBjz38OsSebRcMKM6LWm3f3i+o7ObwOgKad6WDDFzIP0cq7gb3m4M+DK0uglz2LJJ08d+xO5J3E6eMNplE+0XuFq5gYImVAWdD///BogUfVigkxI9XhZysiI20XNe2Ch0VGB+RxYRvpHo3N2GKw6TPWP07yjskxzyYLtE5z363ZZUyQqLRBbxWa4SW4yRDiLGnGjiY/AHXtIGXg3fW0bHG7o+3tqOjxejnF984uAZCWbhUX8NumhhSpmrFGs0+t3WWwkfbvPFWqg0QlOJGsFUWgvtBQHQkwCWU0arkaKzKL6UGiz3Jb4SZWgR+eLHBc7LG1dE2Ml5zuMx21TPDatzmsUdwfVMrzYlBWPT6YL/HZ5IqunTL4m81HAICvt8ccLA/gjj9zphyWTBJSGNctf7c8kIJIGWC9gI8TuzAi6TB1YKKSzg0RqGI//eOTW/6uTVNCiIBNdHsytyU/O6Nxh4eTDW+ovl4dM2IBgNOJ8oWk6iVuIjGyKGIwe7/PESAgYuVdTzp0w+bwuoIYemCVR8iQHL4yx71ipUmj6qM7l99ohFG6t2hlqrIEGC3S0x62VPzP0EkmvRz2c7tVmbTnIN9hPx401SLJdCQAD44f9I1kCZBsqdriKraXwCvagCgR4E9Tn1AdoAhm7LmCbSc5/uLtM74Xs3GLcXznP5+/xde7Y/zqipzYzHowLgbEJhG3pKFqOjunFXFcdkw4C52C2WnYOm4sljnyuCh5RYkzw5B9ClU344D+KP75zNBzFT2bZSfhHkTSW+7QOAH9NX1fvpLoh82sA6IqjzSqB/pT4ZFkOF6QvemB3nfovQorqMoHoG8z5BvBpK7hmPkL3+Fx31O9H/fjftyP+3E/fk3jvaxURYSVQhZSOHAnadc2joL4gCQ0P5BcwEmoeY/qwAhh6FmN6z2UCHi7IRi2yg2+e0rV3Djr8Gl9DQD4sLiGCxIveoIO/89X38ciylp0aSGV56okBMHw2z+8ekQpLPF4zs7X+HBCPUQbJCemvNlPkUvHuaCPp2v2vX29nVFPkoOVU39IZB7nZ9Sj1THD81n8/GVfMfN43RaYVB17kQLA10ti2W53JWyvuFd6mNojSwtEZPLR6Qrn9QY//vJD+h9G3O1j68CsTT02mESmsVYedezfPh6tcNFM2FNZyoAHs+Qv/M3miEO/F4sx9yuromd2rmk1hEpG8yGADfTnZYOzMvkHP9+d4PWWjPhXTYlcWxzXdFzbPm25F9sUXD+uW3Qm4z7qeNQyTHyzGpFhwvB8lWCDECGT325ZGoyKHkfRIWpncryMrGa3yiB7SSHgANTM4OOH9Jx9b3qJpanwKoau15nBJFZO67ZEcztodCLky5VxYqbLzENnlvkAxqhkJlFbIAajoKeQCh8hSCeIzQwQK1vr1NuV88BM4/6mhAiCIH8AXkgUFxHNWCmIWPHgqIcYW76HIQgyfQDoufGAiNWv1P4O89hH1q7oBTCxbEKvF5r7rWbmKWFpaDmEVDEjANhoZNvk6sWm+TYZFtjbErdNhixW+a3ReDSh9+nvbp7g1atjhq+FTxB1DNnh4wgqwL2gC3sdRgnBGXlkKwUZ/ZCDDCn1JoCrXTtx8HVIIeVHjlnNsBKhUcjO6N09nu04bEMAuPy7BwzJBgnOYQWSVAgipgllaQ4ZOANmWUKE5MMs1yr5NZ/1QDPA3IAtgZQpK+gkDirgd3W8l4tqyNIDMdgUyl5QEz+L/YWzDqMRPXhNk8PeRIN4APmsZbivt4rTSUZ5j73J2Ag9k57JQ98dXeJpTrCdhMfPmif4P77+AR1PAI7jRF4oh86lPtt6X2IbXZl8q6BmPfcDP5tfQ0cpRWs0TySVNvBBYJwPRvwCWxMXHhHw4NGSJ1cghl4DmNQt9wIB6ld+vY6TdxCcmlJmFp/MbrCLbkVbUyRt7jq/swmZ1C33K02R+oKvL+d4tT9N8WCVw9kDkng4L3B7OUUeX9TZuOEX/6RK8RgXzQSLfcU/+08fv8RVR5Dst6sZpmWHxaAhbhTKY7qfuXZourQIHrrzTEcdX7d5Tvf1n5YU/ny9r6GGEITc4LTeMeQegmA7QwGwjnSzreCsRBknWuNU0qI6BVlb1o5K7Xl29VZADTII6dEZjStHn79YjeBvBhkUQZZDX/xotkOh6B7+0+oBzqotPpvSItu4DH/3lvTKzdcTyEFbKumgB7vNojQM8RYluX8d2kkO8LiQIfV6AwDtWfOb5RZZ3DxkyqHtMybjdSaDeRuZMpUDygNNcpAk7QB9noqtg7OjDdo+S+k5APdGvZGUBjFocFc53AHBhrWzI+rr5reRZNgLhkxD6YFWpY2dS4Qpso4CuzupieHjdXvN8LLaS8h1BlfStNqcd/jZjto6fp0hWynuQ0qbbP/8xKV+sJXQawU/JPBUnuU7aqFJVy+GBV3AxJ6nmx1k0KmAetbATOLie1sixOarKh1GxzuWU902Na7ic2tWBbKQZC/CJmKRONg89LPAMO4wTORGZLck2Rk2Ky5HChxokn2pV4AfhzsG+kEA3uGdH/fw7/24H/fjftyP+/FrGu9lpToMyjeMlZIgFhtXTlcFNoOjSmUweZrM6YvM4rdOX9OfpcVlrI7+4e0j9L3GZw+J2PKkXuE4pwr0d+qv8TLCvX90/QN8eXvK8WhHJxuuwB6UG1y0E3x5SYSGblWmsPHSITjBgvIXmyNmJSsR2EDexPDETSSlGKdShqeTmE/3XM18szhiqLnODFd9m65ArhyTnZzVd+Qaz9fH7CS1WdR3XF+E9pCxUti1eQpqf1EzQzGcW6iZYXbx984v8Ysrcl5qbiuoscE0IgWHFeHzm2OOPXtyusRmW7En8UU7YUbr09kKndX8b/N5x9Dwqkni+mrckRfqEOJuNEPUPgh8u5tDxlLkuG748yttYJxCESv7TVuw69DJfMtwvg+C5FOx0uva9MqFEP14B6cagAlBPmiunpsuh5Q+hYPvMjbDePhgid4qRgoW6xq3S4IOs9xCngQsQMfy9fUx+0OHkWNUxuoAlI7NPNo+42sqBGW88jHKhFUGL5LkIhDJhB2KMgcfn5fOSyjlmazVvB4nuZQVlGU8IEcnPVfCj47WWEdo/2oxodZHZBcbnyWTlXUGaVPs2GFEXKiSLEQtNKoLyeHmzROL4ji53nfbIrFwDwgzduaQzTpUg4lGr1CP6PqUJxbLTURpXAVXpxi0cJsz8VFHRvKQSSp8ciRS15qvh1fkszvA+dlCQQ3PjCCYfoBh3dTcMU8ZxzCJZl+g2RaMgIixQV6meMTH4xW3MW5/csaRcaIO8BnQ18NnelSvB7g6+fv6s55aBJdxTik9S6SyjYAvgDZ6Hguf3NBggRDba25myKxiaCVU5GDl9btfqr6Xi2qQkQ4ekHqLEWYpXkaT+A7YFZGVWafFJtcOTydLPC2p12iCwrd7gk9HZY/ffPQaT0pym966Alm0xvvL3af48+uPARDk0vcak/gSfOf4mifrm26El8s5ukFq0sgE6/QSonSYxT5eZzUzgWejBpe7xEbNlGM6/2ZfMBPz9GSDUd7jF28o3zMEgfmUjsN4yZNYCAKZcgwpS5GYwLlyWDUlthfx+wRlaQLEntWZQxZfjrbLYC9p0gkTBxslLlllUJQGD6e00P3s1SPW+xbzFlp71r42NsPbaCoOEMQJkEuSbTSqOkHZQ195Z3J89c05v7THj7fsZmOc4kVfSh9dsOjfnU22bCn4Zj9FoSyHHbRWY1bQQj84SN3u6dy2uxLTCV3H23WdzjkPwMSwjZ/MkmG/WRWkSYyQYyiSb54uDeYTus/OC2y2FeQATY4Nuwd1RkPJwItg12n4CEBp7bE3Ga5XdJ/6fQYVoUQnklsOVEA16bDaJGiVYWgnIFXgxR4AS4KEDMnkPwDQAVkMs5+OWtbO9lah7TO2hpQuSWUwOUg+iuMoXse9ybBc0AaBoEOZdKU2htSDNsYHqh9yQ6sHN3kPdRM12m8kXA40H9LKNjpu0Ef42lkFsVd3s3oHt6LSQWmPz06GXvUFv69/e/sBvyM3XsJdlaknKcAwrt4LZNvEmvU5OA3GjgLyZVyUekqEGRZ3YZLOdv+Bg5j1rGt2u4yh7Q8e32Ian80XYg5rD6RaVuI/eUryvde7GX78jx8zY1kGoH0YlQ1D8EG0NKxepXbN/pmDOortgcyh22ecKZutNHJCk2GmCdame5FabWLWI0TXp6AE1HEPNzinOUG46b8A7PS9XFRVJyAFhV0Pwm03ph34YLe1+9jgs8/eAgAkAusff3TyBv/Twz/ClaNJ899vf8B9zXnVoFIGLj4ZlTLYxm3xsq2wjtXWrs1JoxlJND+7fIhRMfTcJDarKlUDlWcP0qMnK/zGySVb5b1ppngTiBQlRQDixOe8RH+w4Dqr8OCMnvon4xWeL495Yjyd7Lga3fU5k2jO6h3WXclyFeclGyTcbEboXo+YMJXNOl6kJnWH1misL+OC6wVQHEycQ0VbUa/6y8uH/BkPH9NGpTNUFZtY6VytxzicNa/ezvjP2Sj5qWrh8fML2ix01xWR0eJ3X19PUE9oUlAHC0TTZihLgx+e0b22QWId5UEmKHROo4z63ONyh42hn226AsttBT9YT2rHi5I3kqK9QDaIwUuUsbLpe53kFIHIGSL2Q3Xm+L6Myp5NPlwQKErD13+zK9n6sZ1keDRf4ziSar65OUIWNyO5tti0BVenKvfwg4ZQBK4OVW3J6COaCkjtWc6jBmODeC+yLJlhBC84jg6lw3h82XObMQAAIABJREFUl2swLKrrTQ27ziEHQ4mQFnS1yGLVFvhcu3G0MwwC+nWURB05hKlhD+7Qq9TX84KiAgd5z4EqI3+dI1/S/9g/DAgPW2R6MOZQfF6+0VCN5IVCBABx0S6f7fGDswt8UNHzmQmHNx09g6OsT++ITXGFAC2GMm5+hAVcATiu4FKvt7iV5HsLIu+offpZPw8cRA4rASdZ9lbOOhxPaIP50fSGeRO7bUmIVrxWHz26wc+u6D1r2gwffnKJ13GT6p3i+QWdIsJmJBn5LEXVhdKxcQu34uMxqxZc/XsdI9zirZGdRIhSHLcryRsYgJq3tIkePqzwtML/C2iq/gvYF9yP+3E/7sf9uB//PMZ7WanKVkBBwIwC3HFMx8gdxGWB7oh2Uk8+vMFZSZZuW1vgv3v41wCA/3b8DwCAl4b6o3+7/AB//y0xKh+frHBebPCsIJejXzYPGBp+s5titaPqtlmVENpDxD7HyWjPfT4hAoKTyWFIAw+eUpX5oCZ5x01HFZFxCo8mBJ/WusdpQVvCZV/hp5ePGII9Gu9ZjtG6DE+maw7tPmQGH1d7nMZztl5hZ3JsYt/0uG7w/ILOGa8qCBXYqEDKwCkp26ZAuy641yOUgxyqHSMRYqVqr0uE0jMaoITHph+MvAOO6gZXG4L+nBOUqgIgOMnVkd9rzMYtTko67y+uz+G+pgq5XgrkSyCPMghTa2w+jp//LFkuTid7HNcN+tiHzqXjnxXKwkMw1KulxywygpdNRVB3vE/9QSJRVlqu3MvcwHrJPVVvZDIS6WU0JBB8LwZnpF2bc697VrWQIuDtglAJs81Tnmf8nsFyMM8tox636xFV0gMrdqeZ3SpHhlONBAg1SJ/pGd70TsA3mk0SugBGdwBwiozSDm2XMQowKTtcRxu+EP8zZKYGFdgkwhfRPWvoIY56bL8gpwJXeYTz+H4WFILu43MA7TnliRu5A8tXB9TPo4zGAbun9NnF492dVKO+15xmIxoK6x6YqgBYyrJ5PcFismEEQ0uPPoYufPnmHCEmxeQbCRGSAUYQlBM6HFsQQL4arsHBASMl59iKepRsjtFJluGE0uPjp1d4NqaK2QWBy4aeianu8NWKeBjBC4xnDYr4/m+6Ah/MqSWVS4svb0/ZNcztNBveCx9hdJmOiR3mlhJu6LU+auH2GjJW6MIf5MYW5FI39KSDCGxP6TMwgmMbDTiB/CQ+g4LMTdxB2Me7Ot7LRdWVAaEMcFN7Z4LG1CGPTknfP7rAhxUtjn84/kd8qAli3HiJ//n29/G/f/U5ACLVHD0kEtO/efAlZnqPn+1okf1qc8ILhRRJmydzxxMaQNKVT4+pX7PqK6zWNcOA01HLzkseApUykJHVYINCzakpBn/6inq2m1WFetJx7/W02qKP/ms7kyOXjn9PC88aVgD8kr64PULfZdyv/NW3Z1BvY79VB/iJQzkeEngCVlH2Y7Y5UfqndH6TqsPtmjYBrlG82OpJi9FBL/RmP8K+S65DF21ynLG95usxnu1Z1qKOPeZlg1/eEMFp+2aM03+i36kWFvU3O8ieNi77D6dY/Ba96KPCMslllBscFXvekLzYHfHEq6XH9X6ULAebGpc3ETZrKMA9pDmaJDEgbeqDMW1O9ibH1WbEJCOOEAQgKwedWYaQvZdYxYWoKHvuYXdW43Y5SvKVg0i8rs3w9e6E48Cmkz1uViO+bgA4SBwippkA7H0MAOvrES1Mw0KnE3QoVEA26WEGskxIP9OFRRWlQ09mK9S6x/OoV75cTGCH411lUF0Kyvalh45We85IZKWFfx4X4KsskYxGjjeXfqchG8XQms99IumIADiBLCZOyV4y8Wf//Q6zox2fq3GK2x0hCDgXbTk35Co10BdIuhL/Ullc72uMDrx7v31D56kucoZugw5wGVKcpAGnwegWEC7AxVQWVySikpl6TuaBAMGgrGUBRPTOfnq+xI+OXuO/mP4cAPBHyx/ip1ET3zmNZezvT6YNHk3XzAU4rbbYR/nbl7en2FyNuYctG5W0+oHsMX0eN5XXij2D+6NAvBMA8qJEuZJQ8REykwA7jbKf4w4CgLsq+TOHRngoPKdP6aPujma7M5ole+/6uId/78f9uB/3437cj1/TeC8r1fCkRaiBuuzRvqCdnrIC0+/d4l8/pBDqT6pr/LAixtyxbLGPW9g/3n0ff33zjHdYv/35l/j9oy8BkKn9//r6tzGJ5gFaJvefvcl4GzzQ28/GtIN+UG0Yfny1mCEvLEN/0wPjeomAjS1w09KuftMVnOW63ZWw0TlGjw2xWCPEszElqki20dJDSY9lNFYvleXKt3UZvrokiNdZhfG4xfqWvktfZrCzSEw6aqFlYDN1ZxOsCxVQjnu+PpfXU67gZOmYtaozhyKzuIz+uW2X3SH9ALib8BOlN7Oq5bzTVVPiZjmG3dB5T77UvOtefFdj8dkMs+fR//hEQlHxiF0Y4cknhAw8HK1xXmxx3cfzFB7lkL7TR5lKrACaNkM4yJ4V2ifoEcB8vov3LPkw32xrNJsSOsoxBhgYiDCrkylwXASWSIUgsNnTbr/d5lQ0DW4/MvB22N0UCKVnR5vVumbXobzuifU9hJQbwfciANhE9rboyfPaR8gxSIHAJusByFPoQvCCfXefHi/x+fwNAOC2r/FqN2fo3OwzDgcXVsDnASF6/0rtmemdv82AUCAkYCIZB7QSIZbgaquoAhwCMJxkIwXVpCoVAOw4wH5CyFJdWDQRAfFeQMqAIt7fdlmifBPZqCKSoQZGdOVI6hPHcd2wSUq7LKEXEfp0YI9d4YF8KbhSVX2qWikZRqA9iTBvlOlgOKX4ro5GLeqiZ3QqU44Z7d+bXuK/mf0UP+/IUOLF7ohlSl9fTFDMqXT8weO3mOgOq0iq8yG1MKxVqI/3aN4MREJwRe5LD5F5ZG9zvq4DrGvnNpnwr8j0v4vn4mrPyVxCBJhFwbC3LwJLjIRJxFDvBdo+4+PPMou6MHAuoQHv6ngvF1W3zhBMhv7bEiEa6v+7f/v/wniFf//2OwCAL9QDZI8jq7dOLj7f9kf47uwS339Mk8kn+SX3V/+3N7+FEATebmmhDkGwoXZrNE+ez44WGGcd92iu2jHerAhWnI0azIoWLxf0Al9tR3gwiX3OIPHF9fmdAOxh4QkAjh9Fhu90DR8E1l3SY9420Qw/63GxHTM8Oc46LOPi8Wo1wzhCssYprBc1p3HYI4ssxo/ZXiPs9YFGEfzC6Zqg1UE3Gbpka6dLcxAo7tD0GTGd45BRg+isgneCIU2dW3x6SlD8sq1wFSUitlfwvYKK5uCuABY/jC/t1ABGwucRbtoDMrrKyOOW5QdaeLxtJzxxTfMG1y19/rBpGTYWMIIcgAAUY4qSG7SA8+me+4nrA+hayoBy3KHdDlIlSu4BANNpqCxNmkVmeZJZb0u210MAUNuUWiPBrMmgArJxn3q2reKFp9/nEMsMarCvmzru4m0WdWKYK1qk2dauEskVSwS4VqMYDwESFt89IR32SbHDix31P79ZHUEKsAQGJoVT+LHD0cM1n1v3YoxiE+0GdYDP0+KT7QRM/AiRJ7mRm9GkrmIMmjACeUy7UQ0taPtHsY/96fogHUqzxWJdWhincHsRowPfpk1YPwskwxmuSaNYsuN7iS9/8Qgy9o9FL6H3aREfeqj5Oi6o8UeuSNZ+3YlH8XiHyZAmtCthb+n9HD9Z4zeinWnrMjQ2Y2305XaM8/iu/ub4JX7SPMOP1x/QZ/QF3NAK8II3pVqQTeN55Ee8bSaYxBbP7PwCP/7iI+5v+8oji2HsJgPEIuOfmWmAPRpovAFZnAuEBfpjRy5UAMpXGbo4FZhOQW9S4AB8cmIKWYCeppQq50S63kHTptK9++Dpe7mo5jcKqlTIfrTCv/vsrwAQTf5/+eo/Q7Onye+//+FfsMb0b5uPcWvpTX/TTvFvj36Ox5rIAj9uPsJfLD4CQJKU42rP5J7eal5Uq8wyfX/dlVi0Fe/qmz7Dg6jXPKu2+HYzv3O8m46O6dZV2DU5t3pMk/HCNp42OI2L/1h3WJuSJ/lVW2KURwKMyfHhbIEymj+82BxxjynXjnNcu4sashe0QwWgxob6pQDkViHUjqUgKvNJPwjSRw7HKCaJzKO1u9PP2mwrfvmECJwegkD+s0Mf9Xy65Y3KelfyJNlvc6iF5ogrOwqUAQnqW3sA3TF9d/MkQEad3ag03EdunUbvNbrYf7RechqPkh7rdcULmKgcHp0v+b70VmNe0Zf7IFhaMcoNy4GsU4DyTGJSVSIxjSdEQBpkKJ3R2MUsV79JaTJyQlISZ9L1Gfqqat6Tv21EDUTmOU0F8f/Z2K8rjhv0g19up7iCkJ2gxKbzPn3+gc2fzB3fz3nV8rPzV2+fcX/bOIW2zRAGCYkTbGAAHbD8es6WoKHyTF7RW4EwSnZ1rki6Rs4UHY4pJN1n1qY4MwDYP/YID+h/WCv5ePPc3TH22C8rqKgB9TlVWQAt7nKr2bN2+ErES+FyIEQ5kl6nVB3hwUkxkIAdgRNazDgA8+GaCrS3JVpPm0g5MXj8KW1OPpre4nm0A22NRmcytPE5OD1KKUN/dP0D/OLmjAlID8Yb3gSvvcSH5ym96Mv1KeuyP5tfswzv//rie4AIqD8iHsj2tobP6DnJlorNGwDAPuqT5MVImFPaVIxP9hgpx5aR7ZN0H9RaUeU6VPwyQEQCXF4ZPhdjFJGdoibWVB7hSMC3B/f8HR3v/rbgftyP+3E/7sf9+Gcy3stK9Q/+q79HPs7xO+MX+FFJfdNMOPzub32Fn7YErex9jk+KCwAUSP2TBW3H/uD0V9i4Cn8dAwr/evmMLb98EHBeYh4DpL+5OMGLHW37styy7GTTFuiNZoPxD+ZL7pu+3MxhnOJEGHUgbwiBLAo52Ft5FFU0GVAplea2qxGC4J3qoTPS08kSW1Nwv7DQlhNWbm7GkJfxeHugf2C4VwcAfjAwr0GsxCHFRwSuupXyqHLDbOCz+RbZUCk4RZUbgOWmgneCe7HhgNGqJ+aOK9PNrmZY6GS6Y4cgGIlsnYTz3VFiTYYgACvghlzNA9btrGqhY2m0MmTKMRyX84KzVp2X8G2CuefzHW6jzKfMDT6YL5HHBmDrUrbtzbZGF/t4o5ryRwfz91HdMYJgvYQPAjfR0NzuNbJoEiG8AGKfyq8z2v4OMisvOOxdKk9WjIPzVebhDljCvnKYP4z2jMsaWMUe1i4ZHZhRQHjSMgx9GJsSgoDSDnnskV8sJxz2LkRIyMYu5/QXAMDYQEaDiuxKoXtg4YbqdKXYZMVG67vh7ohD6c1OMfyLQHKPwZhA71K/z5UBbuy4QqC+abwvfYa+0/zhQqVkF68DZDvISegghj6t3gtmAts6QK8UdPxu1aZ/FxQ5IA2f4YrA0pOQe4jVAOGTCUpxQsjGBydLbuP8+fOPiRmP6GZ01mEa3bSul2Psa/qMB5MtlAicifvF6weM5nz84AafTIgn8PXmBK+u5ziZE/z71eoEF9/SfCVyD11bZpY3leUWiWoFggS608idqAwHH2QbATOn898uagjluV1TzFp0sb3hRmTTOMh0QuaBCIH32/zAOQIQrUzh5rFtdMimf1fHe7moPimXKMsMv12+wIPIec8EcO08VMR1ZqrBn26+CwB4uT/C7x4TgWnjSjzfn+IfFw8AAKtdxRO+Uh6t0dhHCFlnKfbM+5TyclQ3KLVhMss3iyM+tq7TqErDD/1pveMF4Fe3J+j3aQKFAE92mfIMrUoRsOoLXuh8EMii69M36yOs9yUmVQxSt4r7YPpNwbNbf0o9LLOKWFDumfACHQAZuJ9otgWqU4Kes9grfXxM8NLtvuLjqjKLfbwGQnr4TZE0clbAxx6N3WTwtUAR/W2bNsMo9qOvFhOEF/Sil3sBvU/ShKBCOsZOATLw4jNMPgC5z9hIDNuZHCEIhsdbq3my2y5rmgTiYrDe1Jxc9OnxNT6fvsE+Ns3+6vpDXEUdqfeJfBQAjq0DKKFlgJqFiDDYMpGfhjgzMTbUtwaAyiErbZK1IPAi1G9yOueh99gpTgIJhcfRozUT1oKTUAcLX3cS7QbnPZRKEC9pBqOkKyYXDQtudWRYo/nLV+cIt1H76+h8ipu4Iak0+pjq0j9zQKtQvYy97zKgnx/Arq1MC6lI+lCa5AfxKS1agwWgz1Kaii89DkPKm1WJLm5ilPYcp9ddV1CNTLIZ4I53sTS0WAMEDQ+kmuJWQLpkMeiK1CcUFjyLttH+cNCVIvcQkxR4fz7d8rX79nbOZMTJuMEmGk6Xxw1ybdli1PcKvqRr+nS0xPV2xNAwALQL2ry+UVNMMno2Xy7mMLsMV9Ftza1z3jwEI+G1582QazX36lVLUXhDL7xfF+zV7TXYEUuuJewowMUF0XdJlw5E28hhk9wTPwIAhCRnJiC2KUSAiL+ntEdRWDh376h0P+7H/bgf9+N+3I843stK9Serp8hsjo+LKzzW3wAAzlSBP2nO8RfrTwBQVucAmebS4m8XBAs/qVdoXMZVp3OSRe4WCsXU4jTCLk2fMQnlN09fYxTdQjqvsTElGzB0VjNcKmWAFIGN26UI+OKS/GzbZQk4ARkhwnrc4SgaPBTKMgnAeYl9l3PF1W1LXLnB61OgGnVcvazWNaVFALAHBANReIJmh229F+w5KnVAllt0sarK5i2Tb5yXqHKDLu7IBcDZpZttxWbgg5n8kGARdICIO99q0kIIYkgCBJ0zm/iqwBD5KixVLC6SnH0RUuVROnz+0WuWErxZTRla2tscl3vaxW/aArl27GjVdBlX+BABcmz47zqz+Dx6BH8yusbC1Pizt2S4sViNECJ6WtbJuEFJj32fMUntkN3YrguIVjGjGJ1M10Mk9ixqgrWHShIA3BD2XDoE5xGaFDg+EHzUxKDtMzS3dO3USsNF2YxTAfk8MX3y3MJEYpOUASdjQh6kCFi3BYaCsdEZ3t7EdkevuJLJNgIhA1egrk45oPI6g7CCjdsxM0ymEq0EBJi4hJACrvVOwMeizBeBAsHjcXQnnk3zRSsRNBly0BemqmlUd1hG9rYwEr5MPwsyOQYNrF0zje/8SnDVKjzJZgZ/2yFfFQCaxw5iICMNpv/xuMpJh2fHRGistIGWjk1YPpws8GJDCNWqKfHgZMXH5bxkAhimYKTn//7Fdxi+H45jqPCb/QR/F1OvQicBJ+DCwNYVXGXjtEeWW5bDZZWBVfTnfq5QXQmUF7EiNRLtWZTUHFtOy/I+okLxu8dP12hjYIe9JPKhi4RBUVuGiZXyzFB2RhLCEqVyWU6onj+QqL2r471cVP/Hh3+KeqLwSXaLSdRt/LiT+A+bzxhq/f70Lf5u8RQA8PX6GP/6McG/Unh8cXvGWsnj6Y41ZVVmIBHYON95iR+efgsA+LS+wsISbNm4HLddzW5LhbYHhuUGUnp8c0MvXN9mDAOKykLIwIHXnx7fwEfMq7EZsqFP2BfR0Hxg3XqUsZ97Pt6itRneLGLqy3WKLPOlh6joIQ8m2tsdOOscHdNmIVMet6sRxlNa0MmZJrJdrcSsbrCK2k7nJFvvLRYVbIQfVUt0eh/Zl2La4/sf0IJ1uRvj6s2MA6+9FxzKne3EHYamHYcUgjyyvPDMjnaY5w3auKhuq5w3Sc+Xx3z/tPTItcW2iZaIVsFGY/kh0aWMofOfnt7gO2OSPrgg8ZeXH3LMmt9kyI6Si9QQw7dsK/S95pSgQ5s8OIHR4w12mzgZ2tTHk42Ej4bxWUGMYXZG6iVvQBAI1uXIQhUgo2tScALNbcUQu5s4dlQaTdo7E9j/f0JrTEzmMRrGKIYqW6NZM4zXJeSwHzjxkH3aJKmdgtimAHP3qIOOmzLTatb38iZCHSx2Oh3H0HuVRsAVISWqqMDnNUC/giVHAWcn1EdebiuITVpsQ22TtriXyU4vWuvl9GvQ+wMpiAZEF7kEAPppgJ279JkDTD9wDCL8a0rFc8Hr9RQnoz0vkDdNSgR6ON1w2pFxCtOyw3GMgiynBtcN9dxXX8/pO6K0B0Fw+hFUgNgO2lmBUDq+PrITcMd0zybjFsZo3tz2B1GEbmbRCJUg/PLAktIleROyQNrm2FpxTsIMjmECCEcGKv+PYVxn1d3nXwSE+K51KicFQHcP/96P+3E/7sf9uB/3I473slLVwiITAb80J/gPliq2n+2fxMBxggX/7NVHGMcK5bcfvmIx9t/fPEZvNc6jIUNrNbpIKqgyg51JJILTcfIcfd6cwkbK4M7maG3GrNirzQh60Cp2GfY3NZMHhAA7ryjtMalb/MYxsZKflkv8/YpYyXuTwYdYzUmPad0yRD0pOzydkL7yYj/By4sjiLfRm1MG9h0VvQQGfWQeq9YBCZUBt5d0rXRlMRk3LOb3XnAQuZABF7dT1gYiCOwG4ZsKrO/zZQAmPcexnY53eL2mz18uRlB18pXdbUpkm6TRtIOxt6BQZ87ijFFqABGwXmyOcFrRfTqrd3izoc+ncPG0I77djPj43VZzNuzwWUcjqjofVSt82xCC8NObR7h6NU9Eq6mBGrxzg2CD+822gusV79y9F/Bxd16f7CFEwGxGVcnSjik/F2SYMPjemk0BuVFgIqwE++gGF/Wmg/mP9qxZVYVDddzcIdINUHbfaxRFzD4tO+y6/A68vIrB2yGIOx6tt6sRzDKSk856uKHw6BSEl+wVC5EIZG5EUXID5AiAEZCgA1XbEQIXO82RYoef4YtAt3fIpW0l62BDHpAftXyex7Md+zX3bcbXRkx7SAkgeljLnqpfAMhXEvk6HR5kMgsJTqA7TprnkAXodTRdUQEuGsggOgYNaECWOSavlaWhfNgYuWY7jfGMnivjFKp4fY/rBo3J8M0tPWf7dcley/UHGzS7nN9J30tmi9suTeXCUbU8QMOuCPws7bYl6X0PKtShrYPMQ/aa0Yd+dNcNiecCK+kzYttr31XJ6WreEww/EMWUJ3IlAHWbsYZXTHu6TsO5RCMI1qq/w+O9XFT/aPUjFC7DRTdlVuzGFGhdhpdLMl54NFvjqBj6Sh5fb6OBtgj43Ucv8XpHfaVcObYitF4iUw6fxBSJ1mV4via3pXnRII+i+a0p0LkUIhyCQF3STNJ0OUItGGLT2rME43S8w++dPoeJuNTf3D5jp6RcWzyoaQG5akbYdzn3Wx+O1riIPcSvv3xA0OIgeq9cyqgMSBBW7hMEDJo8dISGi9Jgs62YUVsVBi4u4FluUWSWZUBtl3HPWTQqSTVGBpNZYsW+WUy5Lzub7yGlT33UvWZnmkODbgBwJwZZnQT2w/c6R3IVG6/Vy+WcjRXK0vCGo2nyOyb3kwdbZrpudyXywrL70kUzxSq6YG2bAmpsUJ6m7NIBUl7vSoZ7lfYoRmTQAABCCDx+QiL9XDk0JsPVTdRk9JKTf1RtOd9UtBGmHPYVIrBNnp06hLFLBhWtYnhQaQcpA7yPchsZIGKLIATBbYvlvroD/XZtxnCvVB7bXYlVZPmOn65xfkQY6dvbKdw6WtrtiME7WPYFiTvZor7RibVuJbNRAcCPLWSELvVWsNSHPic+jw4kVRnIwFlgMwmcNpAyoC7pPmXKkS0oCDqPayNU5uDe1HwdbR2YTZxtgX4KNhIJEnCTuAE5CrBjxwup3At+Bn0m2HmJ/h6AaNIfSsMbw+2ywg5lyhPOPbuhzSZbyIOHelq0KOKm4MnjlyjiHPViewR15O9wPdgo/8050CR5kC8CbxjkWctMfU76iYtsVlpm+NffaEhDBhYAwd6DNSk5YMTP0LFvP/g7FA5ZcbiJCcjj/GV6DTn0gX1qRvtdBtGLZBJREsPd23cf/n0vF9VfbM6hfXEnnaVUFtYrfCdasF03Y/z8mghCfa/x6Ii2sZ8fv8FVO2ZNqBSB+yH7tsB8vMfFnnajjc14ki+14RdnIBINr1GZG+yaFHt2fpJkENZJ/JdPfwGAtLR/evXJHRu8gQSkao+LffTR7TOcjnd4UNHkd9vVuFhN+Hd86VNwOJD6ppVL3rBOIJ+k8HGl/B15kO0UL6rrTcX2g1IGaOU5oLo3OqWrqMAvcznuUWjHmtBx3SKLE/m+z5DJwH1mFA5hqOAOem/ypEdd9pjHSnK5q0izCbIAvFxMcDGEMR941novmVgBUA9qkL1I6bGNnrtFafBklggkq77khbPI7J3+UNPlvHD6IDCJ/eZx2ZGuMN6yadnhch11qVZykgwQCSXjGIg+3+L6ehKPTyMosOuQ3gv0UQ5TnDSwRsMN0XXne3acmlYt9l1CTqo8BZ2v9hW5RSHqn2XghRQB7J4legl11OHBp6SBbPsMb64jUWmv7/TthKMe9zAOI+JEL4HDRTY+f8ILyLVmwpO0acHqjj33lY/P15ACTOhzr2uWYMFKdL1itMEHwRtR02ru27lFTn3BIT1uI6HjotpPKXTbDxrWKjkjyR6oX2rWpkKknwFgpCe71ciXkvW4fj3GsHfI4qbAR5lYVlreVK/aEpPovPa92SU+H71iN7e/Wn+Mv7kgbsdvnb/GLGvwi3UkLroMLh5U8GBZSwhE4gvRISv4ZKcZrAREQBb17WZdoP4mXreMEmcG1yo38kB0TRMCrEMOAoQODd+tA28iQ0tStoEcBwAY7lN1sBsOhK7wxmhiaNN12HN9R8e7X2vfj/txP+7H/bgf/0zGe1mpCkEV5sNyjakmyOiqH6P3Cl/E6tQ6yXKYo8kePzp6DQDY2QLXzZjTRJZNif2B9MN5yVXsYl8xlPgP+0eoIws2BHLtGarAzbbCOJoKaOVhnUIXoaH/+oMv8BsVmff/vHmE788vODe1aVPyza5L7NbPz97iON/jl2vKGf3m5gjdIpnro/DMlDz03BUqOemUdQ8hEltVSw8df6e3CtXpBsuY2xm8gI8V3H6docszFnwf5DB8DIN5AAAgAElEQVQDucdoThVclRu0RjMENq46LCPcq7VnxyWAUk2GBBV4wT2rp2cL/ODoLb7axGQdL3HTJdjVOcmJLSEAVkbDeC8Ynsq0g5KeJQxNn/HPZlWL82qDNsJ5277g4O1uXUBVyVfWGsWMykPXpG1EFYbK/XI9RjdUyYF60D5KmsTEYDym52C5rpNMRpKcRgwJMDOPLMphTKcRgkA5S6jL0Fvcdzk6o3Ec5TGFtlgO0qFdnvpqIUqaIsSrrACKZKwQgsB17A3aRnM4uPj/2juzHkmS7DofW3yNNfes6qquqq6Z7hlKJEciRQmCXgQJ+iH6iwL0JgLEEKQoURApztIznOnu6tpyj4zFV9v4YObXs/ggQEQIZKHsAwbdPRkZ6e7h4dfs3nPPVYx2zzZ1SHYcPBwGA6je6lIH1jHYoXafGioryPfpB4YDNnXon4Za71FFJYxtl6JqU/KRVlNDg9NdI8AbARPqsg1PKSsBADyULSzztcZkNZQ0gO4wZGJ6wHCAh3vOCkfpa1mxD+5j0Y5tP92RA9+NteJ+aUlRnGwYqZfbEwdbGFJEs7cJ2pPB6H9scfvF6hy/vD/Dd7/x5jKi5pBPvTbjp9+8hGkluXNluYIO97eQFvooPF9a4U926BqYKPJkBvNZosFlSqwlaRRcAjCFsUacPchcOeZrqYC/bxsOW4xZrQ8GxT9wR2OZBRsyIGycQMUY0K8zIPwtzhCeN/jo+SSD6nG2Q5qn+K46IpHR1W6KPFUkThLMUS3t+eSOanPfVwfgzOE6pPDUg/RdkfVIhaEHl7WMrPwOJzXVSTgcUpmS3d7Z4QbTMC6OM4dVW+Bfnnr7xB8Wl3ivlvSzP/n+JaU4l/MaJtzMdZsiC8Hgrivx1xeP6djUJqUbnZUGTFhKyToHGtSe5hpJEEzkicYk7SlFVfUppQ4naY+7uqAA8HABkkiD29WURoc5w5CHwe9JovFo5lPSk6TDm+2SRupJYaiurIyAs5y+0GaTUssIX/b4wSPf1vKfn/wpcqbwX+y/8J/N3QFZ9DFpIYQdhTSO0QJBCItJ+Ft1lwQryPH+GERoAPD13RmN16vqDKYKATBcs6H2rUNwA3z6Uavxvkilwf3QetMJut5cOJibjByb8kmPKrhxDfVU/0L4HspQ32KTsc9waOdow/VJCuXFOOE4Hi03NLru/c2CHlpcOLhBoLJJkL0Vvh4I71TEzvxne7bc4fpuBrMda6cPx90NaVze+4CahTRue+zgQmBmPYM9UECw4pM3GQVjWG9Wb8OQazvVODjx98hB2eB6Fwauh3aMOkw14jtBugBI5wef34c05k6A9FIPg6H2QjZKUbsxmCNkHofULbOU3YSae8P/dDUGDj1kNxloYk3/WY8k1zR9Rs8xiugsg9gKes/+xGAW7CM5A96FMoV5VyJdc7BgFVh8saGFee0YeNlTWURrjiwsHqZ5h2147lRt6XUTIdVsNSehkusE2puCUuwCIAN9poH+VEOEmjxnDocLH9Cvb2ewocWNbyVc+uAL03NKPbPEwjmf3gZ8TX7oTbWO0aI9STWw6MhBzGoGe1fANh9/VI3p30gkEolE9sQnuVP92c0jiDqDFBb3G6+ezXKFbZOTE87Lgxs8K71Ks7MS3+2O6Pff3CwpGyST0Sj/sKixaosP0sHDDi57oA4FfBP9sHPSltM4rUqneDG/w3nmhVH/a/scXVBPfH13hoNJg+NjLxr59fUJ7UYdxlX5u80cqTSoV2E5LR3NQmXc+Tmeg5I3U7RrWzwYiN5qibuqpDRm/8D1Kc8V5nmHx9M1Hf+q89fx3d3CpzTDDqA8aDAv/a7n2XyFi8qvyG/qErs6xyIIhKoupTYXbR0Yc9BBlZhdSvSH/meLeYV/f+KFW0te47U6wlm4VkXW0w6uKHrvlBRaQ7QSo0FFLygFm+UKUlja8R9PK0rF3ewmXpRFaS8GEVoYkjAPsgnGDZTigvfzJWW3ZejaFDxkLORsnH1qVhlcaZDOgvJ7k5NxAO/Hgc68Z7CFA8JOwfWcUrDDLNshIzItO2p/GYYqDM5UedGjCQMe3C6BCPNTk61PU3bHYUTXWU1q7svLBdBzUriKhlG60C407XiMY1Dz0fGIG9A9YEvv/Tuka/XEji1AE+1NLcLv5fOOfG/f3C6p1azdZSSCAQCcdqO/rP9oYM7HlC8N1H6XkYvSICjK3wc3qn40eNClC56+Ycf1wNlJ1MwPTg/v2Zw6Sn3yhsM+9t//Z+d3uNpMocOtkDwYEecEoOfjMG/0HNtbvwtX8w76xt+njAP9DxtSDWst0IQ2PQZ/vw6KL+e8GA3wz5P63TScF4Nd6HHghbSj+tcysMIAYQax6B54KC8NwB25dZ19dod2yLgw0G7Xptbv8AcBkmOUuZGJgXNsFBn+PeHRoLQntfkwAKMTgHQfiNs+Vj7JoNp0CYQIas3wodZVhsNlhZOJT/39weIVDoVPffyifkzK2m2Tg3NLaRfGHJmxb/oM2zonJakyAtug6p1lHVn33e9KSGnw9Mj3jmZCk2UhZxbLtCGj9vfNHN+HNp9l0eK03OLV2rf3GMPHlKNmMMM0mKoEthKYhlTutCfj/d02h0zGNhpjOO5D4LnfFjic13Rei6KlVHbfSapTPT9YoVKj1V+rJerQs9rXCZJcI136L7sQliwXX2+XePfWHzs0Ay81VqEuO520VEfuewm1SZFfhLaRwlEf4zzvaLbtN/0papvhm+oYALDeTMbh1L1EVeWwavxyDw+WJNekkBXcYr0rKLhbx3C78wsEa7mfwPNgKPzwkHcO0G0yTtdhjIzbH5YEGAPKyTgBZrMryECfOWB2XKEfXt9x8G58+NGw67nxwWtICXM3mqCXGlZxOu/NtqQ6b5YrPx84/Kx5MxvdqBjIus4mDt2ppfulu8/Jkk7WHMyMqVCTO6qNwjJ6HQCYwo31xNVojWe0gDvtqOTAakGtMlxa8GRMoxvDoPpxCk69CblJaZEdNpRKVNuxX1NOFEShaFGZpwrbsJAwhaPUpFwLb9Ifgk2/HJ/5jvvrOvw39UXDDzBnFtChfQqW0bB0s9A4P1nT8VrLkK7872a3YxtUe+TgJpoG1Pc8oXp/e5eTpWN21mJajPXx9S73pveAT123khZwTDjcDN+ZXQoZFj6m9O0ugz0gALgwMQhTDddzJOG1ejoqkqGYb3cKCvSEW2zNaF2pd6H0YRhcYcc6M3dIc/87Ulq0bQKZjC2Bw73ZNin6QdvBvfNXHq6HyTn6++wDi8mPlU8yqBaZgsg4zo+3UGEXWEgF6xh+b/EWgJ9SM9Qyr/vp2Eye+LaEwfhASkPB8qSs8Nl0je83vnG76RJ8cXILAHhUbPDnYZh53/mxb49K/2Wcyw7fVT7YDDvFt43/25VKMQkCp3nW4q9fPyEbPZkZ8GGayKzDLtTjUEm4icHswAdIwRzV6kRiYLQAD0a4XFiYYcC1cLRLS6UmqzrA12/T8LderQ6QSkO9ntrwcVesOZRhyI/9MX++vKcd+rouxj495pvHRem/jMP1BID+okR+I0YRTOZIiPN0uiKDDgGLjS7wy5uzcC4G6QN7NKMdiToA4HHwVy2kwi6s/o3l4MyRyGtYRAC+vabvElpVT0INGQCUCg+38CQXD0auMTEaMDBhoZSEGo5pl5DFYDlv4dxonIHEwqZhhystxHxQuQiwxJI9JeeOfIxVLylAAYDpOUzYfqkm8bvMXRAW8XHINyyQBt9bNfUPSBpu/gDHAPAxqNrCjYHUgGwGbW69d3F4PndHdmw1mfs64LALZ4YB4dzSTINzSzvovOwp6BnDUB76e7jMFOouQR/s8HihkebjwtY5hkVoreqUHPtnewYe6nS6tNBLSzVhsRMkrOKGweSjB3F3bMZdpeKQ9xLyNtT/5GiYIiYat2FheLrc+SxQ+Dj7JahXVC8snn12O9oUViU2QZiXJAbnS59t+cH8Bq+rJb678c8Dtc3GcWmp9f8e6pAOgD3wJ0CTcRB6SDnI2hOOUf8zSyyQAHoW7gnNkFz69zO5g50alBP/nje7CZpwvd1dSjHUFRYQ43eLMZDosu8krObow2K2s4z0HDwz5FvOOOD9rMPvbVOk1/IDU4qPlVhTjUQikUhkT3z8y4J/AD85eYt0mmIiO1Tar5DfN3McZhXKMPF6Z3L8j7CzvK4npAAV3GG9y8n44GRakSsTAPRW0mt/dHZFhvd//uY52rDqOzjY4avDa7wo/S72++aQDLa3TY636wW1YACgCSG/vjiBA5CG3V2eKdpZrq5mVNCaPt7iINidAd7hZ8hrWSPA2Dhf9PPDFe3aAJA7VCEVGp1AhO2icZxMLvzfNGP9tU5pjikrNT4/v8PLua/7bnWG18Ou+2IyGqhzB5ZYvDj116AzEq++8y1ASUg59sEWjp22eHzgd5n3/WhEXgiFxiRkB1k1Ka18i1T5umzYTf/w9JoGiq+6kiwi76sCUlhsguGDEJZqwFWXgguLJKzylZK0Q5TSopj0VJu1hpEVoQOAkF5jEwPnMBrqpxb51O8EmjolQ3z/JgwupGB5YmnqzRfPrvyM3HZ0cxpqx8ZwcOF8qwsAvpHjbNLBKGOoL/aMzA24HifK4KgD56BUuev5OMg7tx+oZJliZF1nZgY8mCfw1qtNVTpmImRQkaoq8YrRUINzU40kXKsk0WjqjMwymjaBHlqOmIOq/L1ZqwlYocm04HBe0WD5pk9gzDjs3ahxwg9XgDocbfjQccihrcWONdX+SPtzG86zZ+DNsNv1dUeqcR/0dP+oXYrjz7yK1zgGt0nJWjH5ao2zmb83T8stJLO4bv0xOsdQlmOa9yj3r/vZ3TluVjMyEoFhKE5qep1WghyQeMvh3gf9RsNIdwDLAIXRgUuMZi9cOHBuoaS/X9Jb4VPkAOxSYXlYUbvgblUCwZwhqTgZXjjhkE76cZdZp2i60fYUlaRaMnOAC+1wznCvB4Df4TvLUIfJOlAcNgHsx2+o9GkG1YQbJNxgq3Jcd6E1xgjUOsVfrb17SWsSGhu2qXOq1zS174EbUk2TpMc0jHQbRjs9mvtUzn1XUCA6W2xxcu7rtTPZYSI7fFt78dOvV6d0bCezHao+peC27cZWlhcnd+iMxE1oM2i7hAQlojCYz/yXT1uOt9dLek/OHdX7EFKdQ93m7XpBbUSHRY1cjHL6Mu/Rhp7bt9sFPcQYczCWjbVAANlhsERcbvFvT77BLszJ+qvrx9ROUpzWJIr68dEF/tX8Fb5r/TX44zdfQqzDwqVmMIUjgcnxosLzqReNHWc7Sv/e9SV6I/B+HdoRlEA/ZMqkHyV1MvfX/Czf0iIAAKXoH3okA94Bang4W8dQFD0F5r6TdB+kqQ+8Q+J1Pm9oKtDNxZzsHg8PKt9fGWqGznIamWc1BxNjrRfckbiknHT4o0ff03FdtjOqnwth6dpnuYLqpbcnhE/TUjrYMr+WGj56M6am1IHB5NQ/yMusx+3d9MHA7gcDqTUDHgQfxx1McDlijfD1OwCT8wrGcHQ8pAt7Dns5PGi9WGkIsu6BR7NfqDBsgmiHSQsZave6F+M0m1KTRzLgF5+DK5bphPceHjx3c41++L37FMXbIOYTvn48BAcrQRdEbny7y9AiJBpOixL1NASQD6b6DMFBUQ+vaX3J4uSf+5avf3f2DV4Hr+iLao5Nm+Fk4q/5omjpe3w8rXDb+n9fbUvoXUIp9sNnK/pb1jHMyxZ34RjMlIO/CSMje4Yk1Hl16eByQzoEGDYKyhoBm4zXX5cOJpRqklyjqjOooW2sFhDhPtBzS1NpEml8aSGInbhiVFZw3Pn7aBg+PlNw4T6wpSEf82HhNLTN2YxhVK993MT0byQSiUQie+KT3Kkax2AcwzfbI0qRnk/8TmZwz1FW4LbyqUatBRkT/M7z73GQ1rBBuaEdpx0t4E2uB6Wnthw/OfPCp8O0pgHFxjFsdY6f35z71xlBKcyDrMaXiyu82o3CpW1w5bmvCqheUkO2LO0HE3KGQdvb66nfbQwr91SPw7Edg+4FbpXfjT06vcdRmN14lFUfnMtOZx/MfRx8eqWw6B60BMnUkNPNNO3wvlvgtvMr78/mG7w88CneWqckBPsP85/jv97/Pv7s6oU/t8sZsmBmbkoHPbEoH/ld5g+WN3hR+nRybVI6RsksXtdLOjfbSJydekV1q71b03BNzs42tFNd9QUptnddhqZLKY3LuCNl8CRVMI6RcppzS4O82zZBkhg8Ob6n8/7t9XE4AQYW2l/qLoHqJXRY/fNCQ4T3z8sefZdAqzH9O6zcf//sHVS4x77fHmLTZlRWaOuUZpNqJWB6ARf+HttJajVhLrSNPFTuBj/Yp2crEmddbabk/wsA4l6Sm5ApnZ9dasa0PQ+tOC5x4GH3WaQK620xpve2gn5HHw3iFH9uRa5I9NL1EqbnHzgW0cQV5pAHFfli0kBwi7ttyNJsMxK9sZ6BH/U4XPr7xTmGu/A5yYrR1JX2yMEUo+MRV6FVBt731jxoEdKlQUamJQZdJ70ZPwAYwEz9uSwfbfB06e+BRicopMKP534u8F9cP8frS79TdZZBJJbahZwbxT3TtMOqDfNUw7nLo9Aq0yd0zzEWXNQCdpuADRN4Ugc9C+l16XzbDMafDcI5MGC2aLDpp8MlBkKWw2bBIWz4LBgoE8HUOF0JzgvwxKn/wuo2obIIl9anedtBNWzBHvvny3zS+lIU4OfQKv6BO4eLLTUfL79cnUP2GXojyMi6NRKcObIYNA/aKF6c3OJfH30HAPjV7gyvdofUApMLjSz0mCorcNeV9LOvlltctj41tFMZuTJxOFxUMzJuP5jV+MMjn+p7387xl5dPkYcHuXUMm2EMl+b4wx98h9PMPzwu2hm119xVJQUGnmtfOxkmjXAH54aeWK86HVK+vZa4qvwXbNPlY2tGm/mxZaG2k2SaHJvawQowvL8xDGdzX1f6yfINDmWFuhzrtF/v/OLh6ewS/2n+MwDA37RP8dOLl7hdhd66SpBLjS0sXG5IKQkArxqfJq50Std7qzKkwtAxnj+9wy4YyG92BeAYzo99sux/3j6j97JgtGCq6wxWcYhkbEMZrg0Q2jrCe0phKYAzBpzMKgrUr1YH48DnB+0wbZ36iRyDAnLi8IMzv0AoZY/31RzvL32qPpsrHIYa3JvdEircg62SaPsEzTaouzsBN6TNHINTnFo8RMco3StaBlM6qFA7FQcdPg+LAMEtrrb+2ve9XyC528FaB1RnGx5ywz/JgQiAyw25N928XkJUHEkIWDZzsI9CWxUHjg+21HZlLSebPKM5RGbGkV+WfdDza8J1G1rT2tvgqNRyag/SEwvOLarWf04nswq3t4d0Lt1BUBNPLJhiPqUNb56vQiBinVepTkOJRmuBNgwV6HoOsRlHojnpUJ74z+mr4yvcd/6YnkzuUQiFbyt/r76Y35Ir28VuBs4cWXECoIlBr1YHpDDPCgU+6dGGem5bp1QSsL3v0x1acXjDIcKt2h8biNCLLqVf5A7BuF9nZKXIz1sIbpFMQytLJSCCCtwghZEO7MEEoSH9ax+31BttWwkIBxdaw5wY071uK722Iyin9XVBi5ibyQOrVAckBy0NbhCrBOk9g+k+/uTpx38GkUgkEon8E+GT3KlmUkNKgU5LbIKi8rCooRxHG9IzCbf40ZEXHMySFv/t/VcAfEr3dLKjOYZ37YR8e3OhME9b+tmv7k9JSTdNe9oZcOYVeMM4uceTNbbaH8ff3p9AcIez0u/8vr4687MoAZSHNW7bCYkajOWUkm3bZDTG577/y5KxuiRHn+ePbjFJetqdFokiT+Jdn6IKu7K2TaA7iTT0Rh7Matq9N00KazkNuf6dsws8Kf0OaGcy3KkJltKnfP5Z+RZlmDqdcYXvld9B/MndD5FLDRu8azlGkQgAwDESEL2rFjif+Gv1WX5PIqiUS6TckIK4UunoWuUYzo/W2PT+tfN0NLlfNyWJXJxhmC4azMLutFWjehvwO9XBoahqU9qdz8sWVZ9SOkv3AnYY8M5G5xhnGCAteGiwP1qOvsLfbw5wfTWHCOrixaQhVal1jPqCWyX9jiXMHHWZHcfMVYkXFQ0OTgzgYdej5g76WCELg+APZjVdg7d3CzoO00rwe4lhrryZWEonQzMgcZQWdT0fBU3CwVTh8+s4uGbQT/zfms4bmtf64uAWq670I/DgHacGocx83oA/GGiwrXN09ThGzCZBfBPShcNO2TFAnYa5sZk3MxmEXK++OYUY+mUPLaVqxY6DWUb+vunTCm0wl2DK774GUVp3UUL0467cHCgaCG6UQLPzv/c3F49wMmQXqiX+4+nX+IPpdwCAtSnxl8ZnSI5LjlwofLHw9+pdV+L9xgvsNquxxJKWCn3PYYaeYe7AMZg9WC9ue9CTqp76611OO7o3lZJo6hQ2mDXI+1HhK6Txxi4zn7qtU432dRgxqMeeUv/hKOgiPFMMgwiuSXaT+B3zQ3X538tg0NzbzMIkD94zZFh4bqDqlM4l2YbszoOXfqx8kkH1+8tD8DIHHPD7z94A8Grfi/WCgtT5Yov73qdqfnN/TA+j88kWFgzb8LAWzFG6FwAu6xnehQdWlimy3rvZTejhXCYKT6b3lDp8s1vi684rgLNE43y6xbudfw+tOaVqVC8xSzoadv7t/dFomuAYEB7qNrWwcgxQ2aTHTz4ba7tv6iVOg3PU++2cDNedY+MEFQDLwx3KYNsoucW6H1sdirLDkzBr9CeLNzTX8bvmCBwOHfe31s/rz3Cc+L91Ijf4i+1Lfw1kj0Yl4OFL5YT74As1OaopFV0mPX5v5o//XbfE6yood9MWn09XqPSYah5SjDrV6LREEVTUiTBUH75bT+gBPJl5t6PNgyEIg0J5kvZQRtBkGms41bM5c34YeT0oJeX4wM/HtLXTHJPDhtLj1jH85tLXXod2kYFtk2HHxlm5Q6q5qxNgm4wDnYUDuwwLi3aodYL+2Z1oep0sNA4GVbgReE2ThfjYhrP2BuniaEjXjnXltklDX8SQvu6hQmsY46NymbcMeF7hNDhyCW7xYu5T7292S6ybnJTSWabGc1MSXZPADorTjo/OdtLBhe/d0F40PPRdYSCD+YODd/Ia2rpExWEO/ef+7LNbvLnx6XXjUkA6ZAt/nnmq0JqQknQALEN34e8RueXQC/9Hn315gUxofHt9RK9Ni2E4vcE2LESXRYuvq3P87tRfOwMGHT6YRvtugquwaG+UxOYiBLNS08zXfp15y79hxjH/8HvBKumvNXxdd2hN8vOOwzWtUmAnka5Di9QDUW13V6AvE6p5irVEuhvvIScd9DN/fcpJhzosHmyVwAWnKAjn70U+BtVhEpCzDHmhqOVLb9JRIp9aGiZhKwleC+TXQafRBDX2WHn5aPkkg6qzDM4wvHxyjUr5m+bbqyMYw1GEndkgdwd8y8GjsFPSTmCZ1jRwnDNLdn2vrw7gDMNk7m/Kps5gwm6OMdAuqlOSanrD6wZnpIpluLxe0Bgxh7F2ebzY4Te3x/RAKvOOBAL2Nh0fus67Iw29fz8+ucQXQejzi80j3LcF7U6nWYdl4V93uZ1SbWe5qHBYNlS/1JajS/3x/9GjV3icrbHS/hxK3uObxveYVjpFbyTVkp9P7iio/rx5gle136nWOsXdakIOPMxxmjqSndYoUoVHM3/Nfzy/wH9feUGTdQzz0M6yUxlebQ5IaPV+O8P2yu/Anz67gXWMdrjrrhhdsBJDIpFEGAjuyPP5y+U13lV+QfN6tUTXJmSht5zXOAx/67qa+kVT+HzrLCMXH6Y4cOPvq+LJDqnU4zSk1cxb7AHgtfB9oOGBpLWg4+LcjpNoGukX8YMAqZZUp3LCP4yGgd3soKddnxAWUhpcvA71RTu2VvCWQ4Z+Uz337TV9cIRKEkMLGqsZnOUowm5Xaw4RgpnZJeChHpf8YIs8VVjmwdXISPyfq0f+PSyHtYymK5lGgIUdLjOAyxz11j5s32GagfWjwIZZwATPXTHR1OfZtAlMl0Lehwf50mB56Bc/AGghtEs1JnlPPeAXbw6p1ig6Bq4EeRfrzzqcnfpFI2cO2z5DH+p/MAz9g/Ys2t2qBLOkwx+3Pqu16zNUwXKx0wLWcvq+Ntt89AXuBORtqDEXzrelhF2gHyo+vNAHPROm7CTLFnk+LnrbcByuFUBmYV/462NWma8ZAxBrAflO0jVWcwv1pf8+WcWBdQJ25e/dXZ5QfyvseI8htUgK5VueALhe0DPKcYZ2k1FWhRsAp8H1iTuYehhn6H2kQxLL9/ZajAH4IybWVCORSCQS2ROf5E51vmggSovbqsR6HczTNcf8oMZ5mPfZ6IRaDrZditfBLOCrwyvc9yU23ahkG2YhTqatN4MPO4zlvKa6oNKCHJWyQuHzwxX9/rWYoA7Kxb5JAObVkgBwVNT0t9ZNjqYe63q9luiG6RbAmDLKDY6PtvjR4SUA4El+jy6083BmkUlNTklCWLxe+fSYFJaGCAOAYBZfzfx7/NnVCxq4/bZeQjBHvsl/Wr3ETTMaJiTCIA1SyV+uz/Czld+xKMupLnt9N4PdJeBDaso58KVftr48ucFJvsNp5q/BrzZnlCqfJy250iTc4NFkg9/c+XTq9nIKERyJMqnxxeyGWntW7bhTLbKePmfOHLTl1Ab1t/cnuAyfp0wMFvMaRxO/Oy2kwuv7IZUYdnnh8zW1BG+H+hPIIEEIiyJVeBcUvtglHyho+VTRUPRhlwr41C8Nev97K3jRM0r36oUBKwxOw67q35x9h3eN32l/e3+E9S6n6TCsZ6Nidmqhl2NNsm1SypbMipb8bIeh1g8ZppjIlQSeNnRNU2nwzZVPkVrLPzivtkqBoBblbnR7soUFBoN2ACyzlJpkzbjm54rBpg5sGBJR9FDDtb8oweQJl0wAAAhhSURBVO3Y/oHM0OdT9aPL1vOjO3xzfYQuKIjFbqztyh9uMS9bKh80dUbzSXdtht3NZDTRnxvIfJxW9KOzKzrOSqf45p2/H7NC0Xe1SBVyqUmd3nBHqlumGWWZmAHQiNGrN7Okh0hSjbYTSMJUozQ1lPJtlKTMRnrQ4qvzK7xZ+/tgfZUjuw2KYeMzG+3nwU/8eFSwd20CvQBsKMnkJw09y4yTlIZm3PkZzeG4eCWAa/8cFdpbOPQHQRuQOCDUdpEZsGGm7pYjvx13qlw7cA2Y/uPfqn6SQVUKP8B6U+WUZn10ssbj6RrXD4JDH9K1k1Th5cKnT2+7Cd4GgQEA3K8nZO+mOwkuHT0M77WgthbdC5TBnu7RYoNF1mAXUs9lqkgcc3qywSTtcVb4h/5G5WTy3tSZFyGFh2/XSaAYrfEGIcGT43t8tbzEXPq0TmcliXs4c0i4oS/cbl2Qg03fJTDhXDa2xDzv8NOLl3SuQ2r47XqB+7bAfajFdm2CL0KbiAWDdYxs13orSLhVq9Sb6sM/FOte0HDw5KDDMtTjGp3gXbXAZeNTyLlQOMlCireeU+rdOobLekbuQtPTCr97+h4A8Ly8hQHH29oHM6UFllP/HtO0Rx2sGbXlMI6RTWHfJePkH8tgLKdUvbGcHuTWMnTrcZoL2CjcsLlDsRzN969XszGgPBS/zAzSxOAgHJd1jFyr7u4nJFCDd4WEDJNTHPdiIgBIlh2+OLvBNPH31h+//iGVB9o6Ba4zyOFvujB2DQBb9tQOAwAHi3FC09evz30PIQCRj/2wgLccHB6M5Vf3NNpwW2fYhEHiAJA+6I1utolvx5iMI+kGnYBzDEZzqimikiRmsVND6Vk4/7/h/Tm30EPPJ/MDyHUwuZ8f1NQ7+sX0Bklouv3pxUvfBzqsVXKLoxd+cXtUVnhzv0R1E8oyD1rS9PsS6ZZBP/efaZaNU46meYdX937Bva1y2F7g6DgszPsEVRiq3qQpOHNQoe0qfZtSVteMZXS4LExkCgs0cZVSz7C1AJtZ2ElogTGc2ric4TSVJssUfntzhO5b//0prkfbye7AQR9qqj/vvl3ABhtBlhlgJ6k9pl1nY+uNYaMYzvpe5XQTjiMd674mC0MbhgA80dSupneJH4cHILvzU4xsyKiLzrc4DTaaHzMx/RuJRCKRyJ74pHaqgwHC/RUDLxhUZ8jBhhU13tQZ2iCcaXqOeRDwvDy4AILt6GozQVtr7O7CilYZFGFlejjxLQt3ddjZGI42mF8zoZCG3ZarG7zbpNBh59cbA9MFUYHt8CS7wC6MwqraBLtr/6e4qAHHaO6o0xyM+9UzlxaHIaX5ZfIOR2oLFYQEnUnwVzePwnmlKNMemfW7ktoY9MFzt5j0eLz073G1neHtuxTHoV0gEQZXQf3YJQ6VFDic+t3A02WDVAVls+zxrLhDG1zF/3Z7irswemvXcUwz/ztFonBXllgrf63U1mGtwra1tFjmDSZBkCSNgVV+Na2qnBSV2y6HsRZQ/nN6PrvGqfU7Zt60eF0f4f3KH8dhdotNGLJ+qSWJxoaxYYnwn81Bqkhc0jcCXe+o3cZajuY+lAsaCbCWnJOc5rREncwaTJxf4d/fF1BXCbgKXsGWQYfUcJp1KNCh3YadgZLog/qRuQoupOxdJSF3AkNi3iYO1vj/Ok5WWK2Ab3YH4aca3TqYrN8CzHQIJmGwqYMNyjDRaSyCkG2RN5iKDr/4rb9HTNvTWC/bOLQ7BlsPyu8OX3zhSwLrtsDNlb8ezhk45ai1ohMAhpnhTvmdS9ixJIWCDd8n1SZglcDgrGBzAzZIfdcJXBDYKAGg7CHDfWZrDd2EHa3iXoO1C/fPrMG588fYboH/vfbK+vXGwdQdXD24OABX3/nP+qrLkWwEENK67KRF/9twHdc99MT/LgAwpTGZ+c+z3wFdSGFOZYMXj+4gwxb0+26J7cp/Zwy80Iz1/m93+ehjDAcaUsAaQL5nGLanrNcICSc/iH2iIMK9ZI2DDiMXWcfJ/L5yOUzqkGz88XbJ+P78FpA7BzskQTSgw53FVgzpHWXE0T7S6LpBqNSMyt1awmkO/qV/VuheQA1dA+sE6AAXdvKsdtQelN5osMb//z0LZYDw9v3Uu3fZLnxP3MebBmbuYz76/0fevHmDp0+f/mMfRiQSiUT+L7x+/RpPnjz5xz6MfxCfVFC11uLdu3eYzWZgDzwnI5FIJPKPj3MO2+0Wjx8/BucfZ3XykwqqkUgkEon8/+TjXApEIpFIJPJPkBhUI5FIJBLZEzGoRiKRSCSyJ2JQjUQikUhkT8SgGolEIpHInohBNRKJRCKRPRGDaiQSiUQieyIG1UgkEolE9kQMqpFIJBKJ7IkYVCORSCQS2RMxqEYikUgksidiUI1EIpFIZE/EoBqJRCKRyJ6IQTUSiUQikT0Rg2okEolEInsiBtVIJBKJRPZEDKqRSCQSieyJGFQjkUgkEtkTMahGIpFIJLInYlCNRCKRSGRPxKAaiUQikcieiEE1EolEIpE9EYNqJBKJRCJ7IgbVSCQSiUT2RAyqkUgkEonsiRhUI5FIJBLZEzGoRiKRSCSyJ2JQjUQikUhkT8SgGolEIpHInohBNRKJRCKRPRGDaiQSiUQieyIG1UgkEolE9kQMqpFIJBKJ7IkYVCORSCQS2RMxqEYikUgksidiUI1EIpFIZE/EoBqJRCKRyJ6IQTUSiUQikT0Rg2okEolEInsiBtVIJBKJRPZEDKqRSCQSieyJGFQjkUgkEtkTMahGIpFIJLInYlCNRCKRSGRPxKAaiUQikcieiEE1EolEIpE9EYNqJBKJRCJ7IgbVSCQSiUT2RAyqkUgkEonsib8DcTFkQHuyL9QAAAAASUVORK5CYII=\n", + "text/plain": [ + "Tile(masked_array(\n", + " data=[[1225, 1244, 1247, ..., 1305, 1245, 1206],\n", + " [1166, 1188, 1190, ..., 1381, 1251, 1193],\n", + " [1156, 1110, 1122, ..., 1248, 1245, 1270],\n", + " ...,\n", + " [1485, 1749, 1761, ..., 1034, 996, 998],\n", + " [1780, 1777, 1663, ..., 1008, 1027, 1174],\n", + " [1728, 1647, 1562, ..., 1189, 1297, 1382]],\n", + " mask=[[False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " ...,\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False]],\n", + " fill_value=32767,\n", + " dtype=int16), int16ud32767)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tile" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also still access the string representation easily." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Tile(dimensions=[256, 256], cell_type=CellType(int16ud32767, 32767), cells=[[1225 1244 1247 ... 1305 1245 1206]\\n [1166 1188 1190 ... 1381 1251 1193]\\n [1156 1110 1122 ... 1248 1245 1270]\\n ...\\n [1485 1749 1761 ... 1034 996 998]\\n [1780 1777 1663 ... 1008 1027 1174]\\n [1728 1647 1562 ... 1189 1297 1382]])'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "str(tile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And access the tile's `cells` member which is a numpy ndarray, or more specifically in this case a numpy.ma.MaskedArray." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "masked_array(\n", + " data=[[1225, 1244, 1247, ..., 1305, 1245, 1206],\n", + " [1166, 1188, 1190, ..., 1381, 1251, 1193],\n", + " [1156, 1110, 1122, ..., 1248, 1245, 1270],\n", + " ...,\n", + " [1485, 1749, 1761, ..., 1034, 996, 998],\n", + " [1780, 1777, 1663, ..., 1008, 1027, 1174],\n", + " [1728, 1647, 1562, ..., 1189, 1297, 1382]],\n", + " mask=[[False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " ...,\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False],\n", + " [False, False, False, ..., False, False, False]],\n", + " fill_value=32767,\n", + " dtype=int16)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tile.cells" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `pandas.DataFrame` example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this output you may like to double-click a cell in the `tile2` column to \"expand\" the rows to full size rendering of the tile image." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
                          \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
                          proj_raster_pathtilecrsext
                          0https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(14455356.755667, -2342509.0947641465, 14573964.811098041, -2223901.039333)
                          1https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(14573964.81109804, -2342509.0947641465, 14692572.866529081, -2223901.039333)
                          2https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(14692572.86652908, -2342509.0947641465, 14811180.921960121, -2223901.039333)
                          3https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF(+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=6371007.181 +b=6371007.181 +units=m ,)(14811180.92196012, -2342509.0947641465, 14929788.977391161, -2223901.039333)
                          \n", + "
                          " + ], + "text/plain": [ + " proj_raster_path \\\n", + "0 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "1 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "2 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "3 https://modis-pds.s3.amazonaws.com/MCD43A4.006... \n", + "\n", + " tile \\\n", + "0 Tile(dimensions=[256, 256], cell_type=CellType... \n", + "1 Tile(dimensions=[257, 256], cell_type=CellType... \n", + "2 Tile(dimensions=[257, 256], cell_type=CellType... \n", + "3 Tile(dimensions=[257, 256], cell_type=CellType... \n", + "\n", + " crs \\\n", + "0 (+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=63... \n", + "1 (+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=63... \n", + "2 (+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=63... \n", + "3 (+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=63... \n", + "\n", + " ext \n", + "0 (14455356.755667, -2342509.0947641465, 1457396... \n", + "1 (14573964.81109804, -2342509.0947641465, 14692... \n", + "2 (14692572.86652908, -2342509.0947641465, 14811... \n", + "3 (14811180.92196012, -2342509.0947641465, 14929... " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_df = df.limit(10).toPandas()\n", + "pandas_df.head(4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You still get the default string representatation of a `pandas.Series`" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "proj_raster_path https://modis-pds.s3.amazonaws.com/MCD43A4.006...\n", + "tile Tile(dimensions=[256, 256], cell_type=CellType...\n", + "crs (+proj=sinu +lon_0=0.0 +x_0=0.0 +y_0=0.0 +a=63...\n", + "ext (15404221.199115321, -2342509.0947641465, 1552...\n", + "Name: 8, dtype: object" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_df.iloc[8]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0 Tile(dimensions=[256, 256], cell_type=CellType...\n", + "1 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "2 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "3 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "4 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "5 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "6 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "7 Tile(dimensions=[257, 256], cell_type=CellType...\n", + "8 Tile(dimensions=[256, 256], cell_type=CellType...\n", + "9 Tile(dimensions=[96, 256], cell_type=CellType(...\n", + "Name: tile, dtype: object" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_df.tile" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And nothing different happens for a `pandas.DataFrame` that doesn't have a `Tile` in it." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
                          \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
                          iataairportcitystatecountrylatlongcnt
                          0ORDChicago O'Hare InternationalChicagoILUSA41.979595-87.90446425129
                          1ATLWilliam B Hartsfield-Atlanta IntlAtlantaGAUSA33.640444-84.42694421925
                          2DFWDallas-Fort Worth InternationalDallas-Fort WorthTXUSA32.895951-97.03720020662
                          3PHXPhoenix Sky Harbor InternationalPhoenixAZUSA33.434167-112.00805617290
                          4DENDenver IntlDenverCOUSA39.858408-104.66700213781
                          5IAHGeorge Bush IntercontinentalHoustonTXUSA29.980472-95.33972213223
                          6SFOSan Francisco InternationalSan FranciscoCAUSA37.619002-122.37484312016
                          7LAXLos Angeles InternationalLos AngelesCAUSA33.942536-118.40807411797
                          8MCOOrlando InternationalOrlandoFLUSA28.428889-81.31602810536
                          9CLTCharlotte/Douglas InternationalCharlotteNCUSA35.214011-80.94312610490
                          \n", + "
                          " + ], + "text/plain": [ + " iata airport city state country \\\n", + "0 ORD Chicago O'Hare International Chicago IL USA \n", + "1 ATL William B Hartsfield-Atlanta Intl Atlanta GA USA \n", + "2 DFW Dallas-Fort Worth International Dallas-Fort Worth TX USA \n", + "3 PHX Phoenix Sky Harbor International Phoenix AZ USA \n", + "4 DEN Denver Intl Denver CO USA \n", + "5 IAH George Bush Intercontinental Houston TX USA \n", + "6 SFO San Francisco International San Francisco CA USA \n", + "7 LAX Los Angeles International Los Angeles CA USA \n", + "8 MCO Orlando International Orlando FL USA \n", + "9 CLT Charlotte/Douglas International Charlotte NC USA \n", + "\n", + " lat long cnt \n", + "0 41.979595 -87.904464 25129 \n", + "1 33.640444 -84.426944 21925 \n", + "2 32.895951 -97.037200 20662 \n", + "3 33.434167 -112.008056 17290 \n", + "4 39.858408 -104.667002 13781 \n", + "5 29.980472 -95.339722 13223 \n", + "6 37.619002 -122.374843 12016 \n", + "7 33.942536 -118.408074 11797 \n", + "8 28.428889 -81.316028 10536 \n", + "9 35.214011 -80.943126 10490 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas\n", + "pandas.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2011_february_us_airport_traffic.csv').head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/version.sbt b/version.sbt index eb883072c..c3945bb4f 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.8.0-RC4" +version in ThisBuild := "0.8.0"