[Core-266] Vector Client (#12417)

GitOrigin-RevId: 6d3315ddd9c1042aa2042bb24c48188f85632a87
descarteslabs · Jan 30, 2024 · 092bb20 · 092bb20
1 parent e0eed27
commit 092bb20
Show file tree

Hide file tree

Showing 24 changed files with 3,756 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -52,6 +52,31 @@ Changelog
   `Function.delete_jobs`. It is still possible to encounter request timeouts with very large numbers of jobs;
   workarounds are now documented in the API documentation for the `Function.delete_jobs` method.
 
+### Vector
+
+- The Vector client library, previously available as the `descarteslabs-vector` package on PyPI, has
+  now been integrated into the Descartes Labs Python Client (this package). It should no longer be
+  installed separately.
+- Visualization support (`ipyleaflet.Map`) is enabled when `ipyleaflet` is available. It is not
+  installed by default, but can be installed manually, or by installing the `descarteslabs` python
+  client with the `viz` extra (e.g. `pip install descarteslabs[viz]`).
+- The Vector package now has a `VectorClient` API client, with the usual support for `get_default_client()`
+  and `set_default_client()`. Most constructors and methods now accept an optional `client=` parameter
+  if you need to use something other than the default client.
+- Configuration is now accomplished using the standard `descarteslabs.config` package. In particular,
+  the `vector_url` setting is used to specify the default Vector host. The `VECTOR_API_HOST` environment
+  variable is no longer consulted.
+- Vector client methods now raise standard `descarteslabs.exceptions` Exception classes rather than
+  the `descarteslabs.vector.vector_exceptions` classes of the old client.
+- The `is_spatial=` parameter previously accepted by many methods and functions is now deprecated
+  and ignored. It is not required because existing type information always determines if an operation
+  is spatial or not. Warnings will be generated if it is used.
+- Be advised that feature upload and download (query) do not currently support or impose any limits,
+  and thus allow operations so large and slow that timeouts or other failures may occur. A future
+  version will implement limits and batching, so that large operations can be supported reliably.
+  Until then, the user may wish to implement their own batching were possible to avoid encountering
+  network limits and timeouts.
+
 ### General
 
 - The old client version v1.12.1 is reaching end of life and will longer be supported as of February 2024.

diff --git a/descarteslabs/config/__init__.py b/descarteslabs/config/__init__.py
@@ -241,7 +241,8 @@ def restore_env():
         try:
             # Make sure we selected an environment!
             assert settings.env_for_dynaconf
-            assert settings.gcp_client or settings.aws_client
+            # default_domain must have been set
+            assert settings.default_domain
         except (AttributeError, KeyError, AssertionError):
             message = f"Client configuration '{os.environ[selector]}' doesn't exist!"
             restore_env()

diff --git a/descarteslabs/config/settings.toml b/descarteslabs/config/settings.toml
@@ -1,51 +1,10 @@
 [default]
-aws_client = false
 domain = "descarteslabs.com"
-gcp_client = false
 log_level = "WARNING"
-
-[gcp-production]
-default_host = "@format platform.{this.DOMAIN}"
-gcp_client = true
-grpc_host = "@format {this.DEFAULT_HOST}"
-grpc_port = "443"
-platform_url = "@format https://{this.DEFAULT_HOST}"
-
-catalog_v2_url = "@format {this.PLATFORM_URL}/metadata/v1/catalog/v2"
-iam_url = "@format https://iam.{this.DOMAIN}"
-metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
-raster_url = "@format {this.PLATFORM_URL}/raster/v2"
-yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
-
-[gcp-prerelease]
-default_host = "@format platform-prerelease.{this.DOMAIN}"
-gcp_client = true
-grpc_host = "@format {this.DEFAULT_HOST}"
-grpc_port = "443"
-platform_url = "@format https://{this.DEFAULT_HOST}"
-
-catalog_v2_url = "@format {this.PLATFORM_URL}/metadata/v1/catalog/v2"
-iam_url = "@format https://iam-prerelease.{this.DOMAIN}"
-metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
-raster_url = "@format {this.PLATFORM_URL}/raster/v2"
-yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
-
-[gcp-stage]
-default_domain = "@format stage.{this.DOMAIN}"
-default_host = "@format platform.{this.DEFAULT_DOMAIN}"
-gcp_client = true
-grpc_host = "@format {this.DEFAULT_HOST}"
-grpc_port = "443"
-platform_url = "@format https://{this.DEFAULT_HOST}"
-
-catalog_v2_url = "@format {this.PLATFORM_URL}/metadata/v1/catalog/v2"
-iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
-metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
-raster_url = "@format {this.PLATFORM_URL}/raster/v2"
-yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
+# default_domain must be overridden
+default_domain = ""
 
 [aws-production]
-aws_client = true
 default_domain = "@format production.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 
@@ -55,10 +14,10 @@ iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
 metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
 raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
+vector_url = "@format {this.PLATFORM_URL}/vector/v1"
 yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
 
 [aws-staging]
-aws_client = true
 default_domain = "@format staging.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 
@@ -68,10 +27,10 @@ iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
 metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
 raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
+vector_url = "@format {this.PLATFORM_URL}/vector/v1"
 yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
 
 [aws-dev]
-aws_client = true
 default_domain = "@format dev.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 
@@ -81,10 +40,10 @@ iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
 metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
 raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
+vector_url = "@format {this.PLATFORM_URL}/vector/v1"
 yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
 
 [aws-freemium]
-aws_client = true
 default_domain = "@format freemium.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 
@@ -95,7 +54,6 @@ raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
 
 [testing]
-aws_client = true
 default_domain = "@format dev.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 testing = true
@@ -106,11 +64,11 @@ iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
 metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
 raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
+vector_url = "@format {this.PLATFORM_URL}/vector/v1"
 yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
 
 [aws-testing]
 # testing is now for aws but keep this until it is removed everywhere
-aws_client = true
 default_domain = "@format dev.aws.{this.DOMAIN}"
 platform_url = "@format https://platform.{this.DEFAULT_DOMAIN}"
 testing = true
@@ -121,4 +79,5 @@ iam_url = "@format https://iam.{this.DEFAULT_DOMAIN}"
 metadata_url = "@format {this.PLATFORM_URL}/metadata/v1"
 raster_url = "@format {this.PLATFORM_URL}/raster/v2"
 usage_url = "@format {this.PLATFORM_URL}/usage/v1"
+vector_url = "@format {this.PLATFORM_URL}/vector/v1"
 yaas_url = "@format {this.PLATFORM_URL}/yaas/v1"
diff --git a/descarteslabs/config/tests/test_config.py b/descarteslabs/config/tests/test_config.py
@@ -67,12 +67,6 @@ def test_select_env_from_settings_file(self):
         )
         self.assertEqual(settings.current_env, os.environ.get("DESCARTESLABS_ENV"))
         self.assertEqual(settings.testing, "hello")
-        self.assertEqual(
-            settings.aws_client, os.environ.get("DESCARTESLABS_ENV") == "testing"
-        )
-        self.assertEqual(
-            settings.gcp_client, os.environ.get("DESCARTESLABS_ENV") == "gcp-testing"
-        )
 
     @patch.dict(os.environ, {"DESCARTESLABS_TESTING": "hello"})
     def test_select_env_override_from_env(self):
@@ -142,120 +136,69 @@ def test_env(self):
         assert s1.env == peek1_env
         assert Settings.env == env
 
-        peek2_env = "gcp-stage"
-        s3 = Settings.peek_settings(peek2_env)
-        assert s3.env == peek2_env
-        assert s2.env == env
-        assert s1.env == peek1_env
-        assert Settings.env == env
-
 
 class VerifyValues(unittest.TestCase):
     configs = {
         "aws-dev": {
-            "AWS_CLIENT": True,
             "CATALOG_V2_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1/catalog/v2",
             "COMPUTE_URL": "https://platform.dev.aws.descarteslabs.com/compute/v1",
-            "GCP_CLIENT": False,
             "IAM_URL": "https://iam.dev.aws.descarteslabs.com",
             "LOG_LEVEL": "WARNING",
             "METADATA_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1",
             "PLATFORM_URL": "https://platform.dev.aws.descarteslabs.com",
             "RASTER_URL": "https://platform.dev.aws.descarteslabs.com/raster/v2",
             "USAGE_URL": "https://platform.dev.aws.descarteslabs.com/usage/v1",
+            "VECTOR_URL": "https://platform.dev.aws.descarteslabs.com/vector/v1",
             "YAAS_URL": "https://platform.dev.aws.descarteslabs.com/yaas/v1",
         },
         "aws-production": {
-            "AWS_CLIENT": True,
             "CATALOG_V2_URL": "https://platform.production.aws.descarteslabs.com/metadata/v1/catalog/v2",
             "COMPUTE_URL": "https://platform.production.aws.descarteslabs.com/compute/v1",
-            "GCP_CLIENT": False,
             "IAM_URL": "https://iam.production.aws.descarteslabs.com",
             "LOG_LEVEL": "WARNING",
             "METADATA_URL": "https://platform.production.aws.descarteslabs.com/metadata/v1",
             "PLATFORM_URL": "https://platform.production.aws.descarteslabs.com",
             "RASTER_URL": "https://platform.production.aws.descarteslabs.com/raster/v2",
             "USAGE_URL": "https://platform.production.aws.descarteslabs.com/usage/v1",
+            "VECTOR_URL": "https://platform.production.aws.descarteslabs.com/vector/v1",
             "YAAS_URL": "https://platform.production.aws.descarteslabs.com/yaas/v1",
         },
         "aws-staging": {
-            "AWS_CLIENT": True,
             "CATALOG_V2_URL": "https://platform.staging.aws.descarteslabs.com/metadata/v1/catalog/v2",
             "COMPUTE_URL": "https://platform.staging.aws.descarteslabs.com/compute/v1",
-            "GCP_CLIENT": False,
             "IAM_URL": "https://iam.staging.aws.descarteslabs.com",
             "LOG_LEVEL": "WARNING",
             "METADATA_URL": "https://platform.staging.aws.descarteslabs.com/metadata/v1",
             "PLATFORM_URL": "https://platform.staging.aws.descarteslabs.com",
             "RASTER_URL": "https://platform.staging.aws.descarteslabs.com/raster/v2",
             "USAGE_URL": "https://platform.staging.aws.descarteslabs.com/usage/v1",
+            "VECTOR_URL": "https://platform.staging.aws.descarteslabs.com/vector/v1",
             "YAAS_URL": "https://platform.staging.aws.descarteslabs.com/yaas/v1",
         },
         "aws-testing": {
-            "AWS_CLIENT": True,
             "CATALOG_V2_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1/catalog/v2",
             "COMPUTE_URL": "https://platform.dev.aws.descarteslabs.com/compute/v1",
-            "GCP_CLIENT": False,
             "IAM_URL": "https://iam.dev.aws.descarteslabs.com",
             "LOG_LEVEL": "WARNING",
             "METADATA_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1",
             "PLATFORM_URL": "https://platform.dev.aws.descarteslabs.com",
             "RASTER_URL": "https://platform.dev.aws.descarteslabs.com/raster/v2",
             "TESTING": True,
             "USAGE_URL": "https://platform.dev.aws.descarteslabs.com/usage/v1",
+            "VECTOR_URL": "https://platform.dev.aws.descarteslabs.com/vector/v1",
             "YAAS_URL": "https://platform.dev.aws.descarteslabs.com/yaas/v1",
         },
-        "gcp-prerelease": {
-            "AWS_CLIENT": False,
-            "CATALOG_V2_URL": "https://platform-prerelease.descarteslabs.com/metadata/v1/catalog/v2",
-            "GCP_CLIENT": True,
-            "GRPC_HOST": "platform-prerelease.descarteslabs.com",
-            "GRPC_PORT": "443",
-            "IAM_URL": "https://iam-prerelease.descarteslabs.com",
-            "LOG_LEVEL": "WARNING",
-            "METADATA_URL": "https://platform-prerelease.descarteslabs.com/metadata/v1",
-            "PLATFORM_URL": "https://platform-prerelease.descarteslabs.com",
-            "RASTER_URL": "https://platform-prerelease.descarteslabs.com/raster/v2",
-            "YAAS_URL": "https://platform-prerelease.descarteslabs.com/yaas/v1",
-        },
-        "gcp-production": {
-            "AWS_CLIENT": False,
-            "CATALOG_V2_URL": "https://platform.descarteslabs.com/metadata/v1/catalog/v2",
-            "GCP_CLIENT": True,
-            "GRPC_HOST": "platform.descarteslabs.com",
-            "GRPC_PORT": "443",
-            "IAM_URL": "https://iam.descarteslabs.com",
-            "LOG_LEVEL": "WARNING",
-            "METADATA_URL": "https://platform.descarteslabs.com/metadata/v1",
-            "PLATFORM_URL": "https://platform.descarteslabs.com",
-            "RASTER_URL": "https://platform.descarteslabs.com/raster/v2",
-            "YAAS_URL": "https://platform.descarteslabs.com/yaas/v1",
-        },
-        "gcp-stage": {
-            "AWS_CLIENT": False,
-            "CATALOG_V2_URL": "https://platform.stage.descarteslabs.com/metadata/v1/catalog/v2",
-            "GCP_CLIENT": True,
-            "GRPC_HOST": "platform.stage.descarteslabs.com",
-            "GRPC_PORT": "443",
-            "IAM_URL": "https://iam.stage.descarteslabs.com",
-            "LOG_LEVEL": "WARNING",
-            "METADATA_URL": "https://platform.stage.descarteslabs.com/metadata/v1",
-            "PLATFORM_URL": "https://platform.stage.descarteslabs.com",
-            "RASTER_URL": "https://platform.stage.descarteslabs.com/raster/v2",
-            "YAAS_URL": "https://platform.stage.descarteslabs.com/yaas/v1",
-        },
         "testing": {
-            "AWS_CLIENT": True,
             "CATALOG_V2_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1/catalog/v2",
             "COMPUTE_URL": "https://platform.dev.aws.descarteslabs.com/compute/v1",
-            "GCP_CLIENT": False,
             "IAM_URL": "https://iam.dev.aws.descarteslabs.com",
             "LOG_LEVEL": "WARNING",
             "METADATA_URL": "https://platform.dev.aws.descarteslabs.com/metadata/v1",
             "PLATFORM_URL": "https://platform.dev.aws.descarteslabs.com",
             "RASTER_URL": "https://platform.dev.aws.descarteslabs.com/raster/v2",
             "TESTING": True,
             "USAGE_URL": "https://platform.dev.aws.descarteslabs.com/usage/v1",
+            "VECTOR_URL": "https://platform.dev.aws.descarteslabs.com/vector/v1",
             "YAAS_URL": "https://platform.dev.aws.descarteslabs.com/yaas/v1",
         },
     }

diff --git a/descarteslabs/core/common/vector/__init__.py b/descarteslabs/core/common/vector/__init__.py
diff --git a/descarteslabs/core/common/vector/models.py b/descarteslabs/core/common/vector/models.py
@@ -0,0 +1,40 @@
+from typing import Any, Dict
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+
+class VectorBaseModel(BaseModel):
+    uuid: str = Field(
+        default_factory=uuid4,
+        json_schema_extra={"primary_key": True},
+    )
+
+
+class PointBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "POINT"})
+
+
+class LineBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "LINESTRING"})
+
+
+class PolygonBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "POLYGON"})
+
+
+class MultiPointBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "MULTIPOINT"})
+
+
+class MultiLineBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "MULTILINESTRING"})
+
+
+class MultiPolygonBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "MULTIPOLYGON"})
+
+
+class GenericFeatureBaseModel(VectorBaseModel):
+    geometry: str = Field(json_schema_extra={"geometry": "GEOMETRY"})
+    properties: Dict[str, Any] = {}