Updated Release Candidate

Breaking changes:

* Promote `singer_sdk.helpers.typing` to `singer_sdk.typing` (#84)
* Improve environment variable parsing logic for arrays (#82)
* OAuth: Do not automatically apply `client_email` if `client_id` is missing (#83)
* Remove DatabaseStream class (moved out to #74)

Summary of other changes:

* Refactor private helper methods into private `helper` modules
* Refactor legacy imports into `singer_sdk.helpers._compat` private module
* Automatically detect loops in `next_page_token`
* Numerous other style and internal refactoring improvements

.gitlab-ci.yml

@@ -76,28 +76,3 @@ publish_to_pypi:
       else
         echo "Skipped. (Running in fork.)"
       fi
-    - |
-      echo "Waiting for PyPi availability..."
-      if [[ "$CI_PROJECT_NAMESPACE" == "meltano" ]]
-      then
-        pwd
-        ls -la
-        export VER=$(poetry version --short)
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        echo "Checking for PyPi availability of version $VER"
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
-        export PIPERR=$(pip install tapdance==$VER 2>&1)
-        if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not found. Giving up. Last message from PyPi was $PIPERR"; exit 1; } fi;
-      else
-        echo "Skipped. (Running in fork.)"
-      fi

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2020 AJ
+   Copyright 2021 Meltano
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,6 +1,4 @@
-# `singer-sdk` - an open framework for building Singer-compliant taps
-
-- _Note: This framework is still in early development and planning phases_
+# `singer-sdk` - a framework for building Singer taps
 
 ## Strategies for Optimized Tap Development
 

cookiecutter/tap-template/README.md

@@ -1,10 +1,3 @@
-_**NOTE:** This framework is still in early exploration and development phases. For more
-information and to be updated on this project, please feel free to subscribe to our
-[original Meltano thread](https://gitlab.com/meltano/meltano/-/issues/2401) and the
-[initial PR for the underlying framework](https://gitlab.com/meltano/tap-base/-/merge_requests/1)._
-
---------------------------------
-
 # Singer SDK Tap Template
 
 To use this cookie cutter template:

cookiecutter/tap-template/cookiecutter.json

@@ -3,7 +3,6 @@
     "tap_id": "tap-{{ cookiecutter.source_name.lower() }}",
     "library_name": "{{ cookiecutter.tap_id.replace('-', '_') }}",
     "stream_type": [
-        "Database",
         "REST",
         "GraphQL",
         "Other"

cookiecutter/tap-template/cookiecutter.tests.yml

@@ -1,35 +1,30 @@
 tests:
-  - source_name: DatabaseTemplateTest
-    tap_id: test-tap-database-type
-    stream_type: Database
-    auth_method": Custom or N/A
-
   - source_name: RESTSimpleAuthTemplateTest
     tap_id: test-tap-rest-simpleauth-type
     stream_type: REST
-    auth_method": Simple
+    auth_method: Simple
 
   - source_name: RESTOAuthTemplateTest
     tap_id: test-tap-rest-oath-type
     stream_type: REST
-    auth_method": OAuth
+    auth_method: OAuth
 
   - source_name: RESTJWTTemplateTest
     tap_id: test-tap-rest-simpleauth-type
     stream_type: REST
-    auth_method": JWT
+    auth_method: JWT
 
   - source_name: RESTJWTTemplateTest
     tap_id: test-tap-rest-simpleauth-type
     stream_type: REST
-    auth_method": Custom or N/A
+    auth_method: Custom or N/A
 
   - source_name: GraphQLJWTTemplateTest
     tap_id: test-tap-graphql-jwt-type
     stream_type: GraphQL
-    auth_method": JWT
+    auth_method: JWT
 
   - source_name: OtherCustomTemplateTest
     tap_id: test-tap-other-custom-type
     stream_type: Other
-    auth_method": Custom or N/A
+    auth_method: Custom or N/A

cookiecutter/tap-template/{{cookiecutter.library_name}}/README.md

@@ -1,13 +1,6 @@
-_**NOTE:** The Singer SDK framework is still in early exploration and development phases. For more
-information and to be updated on this project, please feel free to subscribe to our
-[original Meltano thread](https://gitlab.com/meltano/meltano/-/issues/2401) and the
-[initial PR for the underlying framework](https://gitlab.com/meltano/singer-sdk/-/merge_requests/1)._
+# {{cookiecutter.tap_id}}
 
---------------------------------
-
-# Welcome to the {{cookiecutter.tap_id}} Singer Tap!
-
-This Singer-compliant tap was created using the [Singer SDK](https://gitlab.com/meltano/singer-sdk).
+This Singer tap was created using the [Singer SDK](https://gitlab.com/meltano/singer-sdk).
 
 ## Getting Started
 

cookiecutter/tap-template/{{cookiecutter.library_name}}/pyproject.toml

@@ -1,17 +1,14 @@
 [tool.poetry]
 name = "{{cookiecutter.tap_id}}"
 version = "0.0.1"
-description = "`{{cookiecutter.tap_id}}` is Singer-compliant {{cookiecutter.source_name}} tap built with Singer SDK."
+description = "`{{cookiecutter.tap_id}}` is Singer tap for {{cookiecutter.source_name}}, built with the Singer SDK."
 authors = ["TODO: Your Name <[email protected]>"]
 license = "Apache v2"
 
 [tool.poetry.dependencies]
 python = "^3.6"
 requests = "^2.25.1"
-# Note: Until we clear the first non-prerelease, `singer-sdk` need to be pinned to a specific version.
-#       For a list of released versions: https://pypi.org/project/singer-sdk/#history
-#       To safely update the version number: `poetry add singer-sdk==0.0.2-dev.1068770959`
-singer-sdk = "0.0.2-dev.1068770959"
+singer-sdk = "^0.1.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.1.2"

cookiecutter/tap-template/{{cookiecutter.library_name}}/{{cookiecutter.library_name}}/streams.py

@@ -12,15 +12,15 @@
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 {% endif %}
 
-{% if cookiecutter.stream_type in ["GraphQL", "REST"] %}
+{% if cookiecutter.stream_type in ("GraphQL", "REST") %}
 from singer_sdk.authenticators import (
     APIAuthenticatorBase,
     SimpleAuthenticator,
     OAuthAuthenticator,
     OAuthJWTAuthenticator
 )
 {% endif %}
-from singer_sdk.helpers.typing import (
+from singer_sdk.typing import (
     ArrayType,
     BooleanType,
     DateTimeType,
@@ -39,20 +39,25 @@
 class {{ cookiecutter.source_name }}Stream(Stream):
     """Stream class for {{ cookiecutter.source_name }} streams."""
 
-    def get_records(self, partition: Optional[dict]) -> Iterable[dict]:
+    def get_records(self, partition: Optional[dict] = None) -> Iterable[dict]:
         """Return a generator of row-type dictionary objects."""
         # TODO: Write logic to extract data from the upstream source.
         # rows = mysource.getall()
         # for row in rows:
         #     yield row.to_dict()
         raise NotImplementedError("The method is not yet implemented (TODO)")
 
-{% endif %}
-{% if cookiecutter.stream_type in ["GraphQL", "REST"] %}
+{% elif cookiecutter.stream_type in ("GraphQL", "REST") %}
 class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):
     """{{ cookiecutter.source_name }} stream class."""
 
-    url_base = "https://api.mysample.com"
+    @property
+    def url_base(self) -> str:
+        """Return the API URL root, configurable via tap settings."""
+        return self.config["api_url"]
+
+    # Alternatively, use a static string for url_base:
+    # url_base = "https://api.mysample.com"
 
 {% if cookiecutter.stream_type == "REST" %}
     def get_url_params(
@@ -66,9 +71,9 @@ def get_url_params(
         logic.
         """
         params = {}
-        starting_datetime = self.get_starting_datetime(partition)
+        starting_datetime = self.get_starting_timestamp(partition)
         if starting_datetime:
-            params.update({"updated": starting_datetime})
+            params["updated"] = starting_datetime
         return params
 
 {% endif %}
@@ -100,10 +105,12 @@ def authenticator(self) -> APIAuthenticatorBase:
 
 
 {% if cookiecutter.stream_type == "GraphQL" %}
-# TODO: - Override `StreamA` and `StreamB` with your own stream definition.
+# TODO: - Override `UsersStream` and `GroupsStream` with your own stream definition.
 #       - Copy-paste as many times as needed to create multiple stream types.
-class StreamA({{ cookiecutter.source_name }}Stream):
+class UsersStream({{ cookiecutter.source_name }}Stream):
     name = "users"
+    # Optionally, you may also use `schema_filepath` in place of `schema`:
+    # schema_filepath = SCHEMAS_DIR / "users.json"
     schema = PropertiesList(
         Property("name", StringType),
         Property("id", StringType),
@@ -137,7 +144,7 @@ class StreamA({{ cookiecutter.source_name }}Stream):
         """
 
 
-class StreamB({{ cookiecutter.source_name }}Stream):
+class GroupsStream({{ cookiecutter.source_name }}Stream):
     name = "groups"
     schema = PropertiesList(
         Property("name", StringType),
@@ -155,16 +162,18 @@ class StreamB({{ cookiecutter.source_name }}Stream):
         """
 
 
-{% elif cookiecutter.stream_type in ["Other", "REST"] %}
-# TODO: - Override `StreamA` and `StreamB` with your own stream definition.
+{% elif cookiecutter.stream_type in ("Other", "REST") %}
+# TODO: - Override `UsersStream` and `GroupsStream` with your own stream definition.
 #       - Copy-paste as many times as needed to create multiple stream types.
-class StreamA({{ cookiecutter.source_name }}Stream):
+class UsersStream({{ cookiecutter.source_name }}Stream):
     name = "users"
-{% if cookiecutter.stream_type in ["REST"] %}
+{% if cookiecutter.stream_type == "REST" %}
     path = "/users"
 {% endif %}
     primary_keys = ["id"]
     replication_key = None
+    # Optionally, you may also use `schema_filepath` in place of `schema`:
+    # schema_filepath = SCHEMAS_DIR / "users.json"
     schema = PropertiesList(
         Property("name", StringType),
         Property("id", StringType),
@@ -177,9 +186,9 @@ class StreamA({{ cookiecutter.source_name }}Stream):
     ).to_dict()
 
 
-class StreamB({{ cookiecutter.source_name }}Stream):
+class GroupsStream({{ cookiecutter.source_name }}Stream):
     name = "groups"
-{% if cookiecutter.stream_type in ["REST"] %}
+{% if cookiecutter.stream_type == "REST" %}
     path = "/groups"
 {% endif %}
     primary_keys = ["id"]
@@ -190,25 +199,3 @@ class StreamB({{ cookiecutter.source_name }}Stream):
         Property("modified", DateTimeType),
     ).to_dict()
 {% endif %}
-
-{% if cookiecutter.stream_type == "Database" %}
-class {{ cookiecutter.source_name }}Stream(DatabaseStream):
-    """Stream class for {{ cookiecutter.source_name }} database streams."""
-
-    @classmethod
-    def execute_query(cls, sql: Union[str, List[str]], config) -> Iterable[dict]:
-        """Run a query in snowflake."""
-        connection = cls.open_connection(config=config)
-        """Connect to database."""
-        # TODO: Define the process of executing a query against your database
-        #       and returning a list or other iterable containing the resulting
-        #       rows.
-        raise NotImplementedError("The method is not yet implemented (TODO)")
-
-    @classmethod
-    def open_connection(cls, config) -> Any:
-        """Connect to database."""
-        # TODO: Define the process of connecting to your database and returning
-        #       a connection object.
-        raise NotImplementedError("The method is not yet implemented (TODO)")
-{% endif %}

cookiecutter/tap-template/{{cookiecutter.library_name}}/{{cookiecutter.library_name}}/tap.py

@@ -4,7 +4,7 @@
 from typing import List
 import click
 from singer_sdk import Tap, Stream
-from singer_sdk.helpers.typing import (
+from singer_sdk.typing import (
     ArrayType,
     BooleanType,
     DateTimeType,
@@ -19,20 +19,18 @@
 # TODO: Import your custom stream types here:
 from {{ cookiecutter.library_name }}.streams import (
     {{ cookiecutter.source_name }}Stream,
-{% if cookiecutter.stream_type in ["GraphQL", "REST", "Other"] %}
-    StreamA,
-    StreamB,
+{% if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
+    UsersStream,
+    GroupsStream,
 {% endif %}
 )
 
-PLUGIN_NAME = "{{ cookiecutter.tap_id }}"
-
-{% if cookiecutter.stream_type in ["GraphQL", "REST", "Other"] %}
+{% if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
 # TODO: Compile a list of custom stream types here
 #       OR rewrite discover_streams() below with your custom logic.
 STREAM_TYPES = [
-    StreamA,
-    StreamB,
+    UsersStream,
+    GroupsStream,
 ]
 {% endif %}
 
@@ -46,10 +44,10 @@ class Tap{{ cookiecutter.source_name }}(Tap):
         Property("auth_token", StringType, required=True),
         Property("project_ids", ArrayType(StringType), required=True),
         Property("start_date", DateTimeType),
-        Property("api_url", StringType),
+        Property("api_url", StringType, default="https://api.mysample.com"),
     ).to_dict()
 
-{% if cookiecutter.stream_type in ["GraphQL", "REST", "Other"] %}
+{% if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
     def discover_streams(self) -> List[Stream]:
         """Return a list of discovered streams."""
         return [stream_class(tap=self) for stream_class in STREAM_TYPES]

cookiecutter/tap-template/{{cookiecutter.library_name}}/{{cookiecutter.library_name}}/tests/__init__.py

@@ -1 +1 @@
-"""Test suite for {{ cookiecutter.library_name }}."""
+"""Test suite for {{ cookiecutter.tap_id }}."""

cookiecutter/tap-template/{{cookiecutter.library_name}}/{{cookiecutter.library_name}}/tests/{{ 'test' }}_core.py

@@ -1,12 +1,13 @@
 """Tests init and discovery features for {{ cookiecutter.tap_id }}."""
 
-from singer_sdk.helpers.util import utc_now
+import datetime
+
 from singer_sdk.helpers.testing import get_basic_tap_test
 
 from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
 
 SAMPLE_CONFIG = {
-    "start_date": utc_now()
+    "start_date": datetime.datetime.now(datetime.timezone.utc)
     # TODO: Initialize minimal tap config and/or register env vars in test harness
 }