From 1cc4fc0555394b74ee27283a4c4320d34d8d4b1d Mon Sep 17 00:00:00 2001 From: Josh lloyd Date: Thu, 14 Mar 2024 16:31:22 -0600 Subject: [PATCH] fixed time entries and project streams --- .github/dependabot.yml | 26 ----- .github/workflows/test.yml | 29 ----- README.md | 1 - tap_toggl/client.py | 19 +-- tap_toggl/streams.py | 232 ++++++++++++++++++++++--------------- tap_toggl/tap.py | 11 +- 6 files changed, 150 insertions(+), 168 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 933e6b1..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,26 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: pip - directory: "/" - schedule: - interval: "daily" - commit-message: - prefix: "chore(deps): " - prefix-development: "chore(deps-dev): " - - package-ecosystem: pip - directory: "/.github/workflows" - schedule: - interval: daily - commit-message: - prefix: "ci: " - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: "weekly" - commit-message: - prefix: "ci: " diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 1d3df60..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -### A CI workflow template that runs linting and python testing - -name: Test tap-toggl - -on: [push] - -jobs: - pytest: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry - - name: Install dependencies - run: | - poetry install - - name: Test with pytest - run: | - poetry run pytest diff --git a/README.md b/README.md index 327f322..0592485 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ tap-toggl --about --format=markdown | api_token | True | None | The token to authenticate against the Toggl API | | detailed_report_trailing_days| False | 1 | Provided for backwards compatibility. Does nothing. | | start_date | False | | The earliest record date to sync. In the format YYYY-MM-DD. | -| user_agent | False | | Inserts a user agent into the request header | | stream_maps | False | None | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). | | stream_map_config | False | None | User-defined config values to be used within map expressions. | | faker_config | False | None | Config for the [`Faker`](https://faker.readthedocs.io/en/master/) instance variable `fake` used within map expressions. Only applicable if the plugin specifies `faker` as an addtional dependency (through the `singer-sdk` `faker` extra or directly). | diff --git a/tap_toggl/client.py b/tap_toggl/client.py index a6e44da..d7d8281 100644 --- a/tap_toggl/client.py +++ b/tap_toggl/client.py @@ -11,11 +11,6 @@ from singer_sdk.pagination import BasePageNumberPaginator, BaseAPIPaginator from singer_sdk.streams import RESTStream -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources - _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] @@ -25,7 +20,7 @@ class TogglStream(RESTStream): @property def url_base(self) -> str: """Return the API URL root, configurable via tap settings.""" - return "https://api.track.toggl.com/api/v9" + return "https://api.track.toggl.com" records_jsonpath = "$[*]" @@ -36,15 +31,13 @@ def http_headers(self) -> dict: Returns: A dictionary of HTTP headers. """ - auth_str = bytes(f"{self.config.get('api_token')}:api_token", 'utf-8') + auth_str = bytes(f"{self.config.get('api_token')}:api_token", "utf-8") encoded_token = b64encode(auth_str).decode("ascii") headers = {"content-type": "application/json"} if "api_token" in self.config: headers["Authorization"] = f"Basic {encoded_token}" else: self.logger.error("No API token provided") - if "user_agent" in self.config: - headers["User-Agent"] = self.config.get("user_agent") return headers def start_time_to_epoch(self, start_time: str) -> int: @@ -73,9 +66,9 @@ def get_new_paginator(self) -> BaseAPIPaginator: return BasePageNumberPaginator(1) def get_url_params( - self, - context: dict | None, # noqa: ARG002 - next_page_token: Any | None, # noqa: ANN401 + self, + context: dict | None, # noqa: ARG002 + next_page_token: Any | None, # noqa: ANN401 ) -> dict[str, Any]: """Return a dictionary of values to be used in URL parameterization. @@ -87,7 +80,7 @@ def get_url_params( A dictionary of URL query parameters. """ params: dict = {} - if self.config.get("start_date"): + if self.config.get("start_date") and self.name != "projects": params["since"] = self.start_time_to_epoch(self.config.get("start_date")) if next_page_token: params["page"] = next_page_token diff --git a/tap_toggl/streams.py b/tap_toggl/streams.py index df41d6f..f26bff7 100644 --- a/tap_toggl/streams.py +++ b/tap_toggl/streams.py @@ -6,20 +6,18 @@ import typing as t from singer_sdk import typing as th +from singer_sdk.pagination import BaseAPIPaginator, SimpleHeaderPaginator from tap_toggl.client import TogglStream, TogglPaginationStream -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources +_TToken = t.TypeVar("_TToken") class ClientsStream(TogglStream): """Define custom stream.""" name = "clients" - path = "/me/clients" + path = "/api/v9/me/clients" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" schema = th.PropertiesList( @@ -36,7 +34,7 @@ class OrganizationsStream(TogglStream): """Define custom stream.""" name = "organizations" - path = "/me/organizations" + path = "/api/v9/me/organizations" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" schema = th.PropertiesList( @@ -55,13 +53,16 @@ class OrganizationsStream(TogglStream): th.Property("pricing_plan_id", th.IntegerType), th.Property("server_deleted_at", th.DateTimeType), th.Property("suspended_at", th.DateTimeType), - th.Property("trial_info", th.ObjectType( - th.Property("last_pricing_plan_id", th.IntegerType), - th.Property("next_payment_date", th.DateTimeType), - th.Property("trial", th.BooleanType), - th.Property("trial_available", th.BooleanType), - th.Property("trial_end_date", th.DateTimeType), - )), + th.Property( + "trial_info", + th.ObjectType( + th.Property("last_pricing_plan_id", th.IntegerType), + th.Property("next_payment_date", th.DateTimeType), + th.Property("trial", th.BooleanType), + th.Property("trial_available", th.BooleanType), + th.Property("trial_end_date", th.DateTimeType), + ), + ), th.Property("user_count", th.IntegerType), ).to_dict() @@ -77,7 +78,7 @@ class GroupsStream(TogglStream): parent_stream_type = OrganizationsStream name = "groups" - path = "/organizations/{organization_id}/groups" + path = "/api/v9/organizations/{organization_id}/groups" primary_keys: t.ClassVar[list[str]] = ["group_id"] replication_key = "at" schema = th.PropertiesList( @@ -86,12 +87,17 @@ class GroupsStream(TogglStream): th.Property("name", th.StringType), th.Property("permissions", th.StringType), th.Property("organization_id", th.IntegerType), - th.Property("users", th.ArrayType(th.ObjectType( - th.Property("avatar_url", th.StringType), - th.Property("joined", th.BooleanType), - th.Property("name", th.StringType), - th.Property("user_id", th.IntegerType), - ))), + th.Property( + "users", + th.ArrayType( + th.ObjectType( + th.Property("avatar_url", th.StringType), + th.Property("joined", th.BooleanType), + th.Property("name", th.StringType), + th.Property("user_id", th.IntegerType), + ) + ), + ), th.Property("workspaces", th.ArrayType(th.IntegerType)), ).to_dict() @@ -101,7 +107,7 @@ class UsersStream(TogglPaginationStream): parent_stream_type = OrganizationsStream name = "users" - path = "/organizations/{organization_id}/users" + path = "/api/v9/organizations/{organization_id}/users" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = None schema = th.PropertiesList( @@ -109,10 +115,15 @@ class UsersStream(TogglPaginationStream): th.Property("avatar_url", th.StringType), th.Property("can_edit_email", th.BooleanType), th.Property("email", th.StringType), - th.Property("groups", th.ArrayType(th.ObjectType( - th.Property("group_id", th.IntegerType), - th.Property("name", th.StringType), - ))), + th.Property( + "groups", + th.ArrayType( + th.ObjectType( + th.Property("group_id", th.IntegerType), + th.Property("name", th.StringType), + ) + ), + ), th.Property("id", th.IntegerType), th.Property("inactive", th.BooleanType), th.Property("invitation_code", th.StringType), @@ -121,19 +132,24 @@ class UsersStream(TogglPaginationStream): th.Property("owner", th.BooleanType), th.Property("organization_id", th.IntegerType), th.Property("user_id", th.IntegerType), - th.Property("workspaces", th.ArrayType(th.ObjectType( - th.Property("admin", th.BooleanType), - th.Property("inactive", th.BooleanType), - th.Property("name", th.StringType), - th.Property("role", th.StringType), - th.Property("workspace_id", th.IntegerType), - ))), + th.Property( + "workspaces", + th.ArrayType( + th.ObjectType( + th.Property("admin", th.BooleanType), + th.Property("inactive", th.BooleanType), + th.Property("name", th.StringType), + th.Property("role", th.StringType), + th.Property("workspace_id", th.IntegerType), + ) + ), + ), ).to_dict() def get_url_params( - self, - context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, # noqa: ANN401 + self, + context: dict | None, # noqa: ARG002 + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -156,7 +172,7 @@ class WorkspacesStream(TogglStream): # parent_stream_type = OrganizationsStream name = "workspaces" - path = "/me/workspaces" + path = "/api/v9/me/workspaces" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" schema = th.PropertiesList( @@ -191,13 +207,16 @@ class WorkspacesStream(TogglStream): th.Property("rounding_minutes", th.IntegerType), th.Property("server_deleted_at", th.DateTimeType), th.Property("suspended_at", th.DateTimeType), - th.Property("te_constraints", th.ObjectType( - th.Property("description_present", th.BooleanType), - th.Property("project_present", th.BooleanType), - th.Property("tag_present", th.BooleanType), - th.Property("task_present", th.BooleanType), - th.Property("time_entry_constraints_enabled", th.BooleanType), - )), + th.Property( + "te_constraints", + th.ObjectType( + th.Property("description_present", th.BooleanType), + th.Property("project_present", th.BooleanType), + th.Property("tag_present", th.BooleanType), + th.Property("task_present", th.BooleanType), + th.Property("time_entry_constraints_enabled", th.BooleanType), + ), + ), th.Property("working_hours_in_minutes", th.IntegerType), ).to_dict() @@ -213,7 +232,8 @@ class ProjectsStream(TogglPaginationStream): parent_stream_type = WorkspacesStream name = "projects" - path = "/workspaces/{workspace_id}/projects" + path = "/api/v9/workspaces/{workspace_id}/projects" + rest_method = "GET" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" schema = th.PropertiesList( @@ -228,10 +248,13 @@ class ProjectsStream(TogglPaginationStream): th.Property("color", th.StringType), th.Property("created_at", th.DateTimeType), th.Property("currency", th.StringType), - th.Property("current_period", th.ObjectType( - th.Property("end_date", th.DateTimeType), - th.Property("start_date", th.DateTimeType), - )), + th.Property( + "current_period", + th.ObjectType( + th.Property("end_date", th.DateTimeType), + th.Property("start_date", th.DateTimeType), + ), + ), th.Property("end_date", th.DateTimeType), th.Property("estimated_hours", th.IntegerType), th.Property("estimated_seconds", th.IntegerType), @@ -243,14 +266,19 @@ class ProjectsStream(TogglPaginationStream): th.Property("rate", th.NumberType), th.Property("rate_last_updated", th.DateTimeType), th.Property("recurring", th.BooleanType), - th.Property("recurring_parameters", th.ArrayType(th.ObjectType( - th.Property("custom_period", th.IntegerType), - th.Property("estimated_seconds", th.IntegerType), - th.Property("parameter_end_date", th.DateTimeType), - th.Property("parameter_start_date", th.DateTimeType), - th.Property("period", th.StringType), - th.Property("project_start_date", th.DateTimeType), - ))), + th.Property( + "recurring_parameters", + th.ArrayType( + th.ObjectType( + th.Property("custom_period", th.IntegerType), + th.Property("estimated_seconds", th.IntegerType), + th.Property("parameter_end_date", th.DateTimeType), + th.Property("parameter_start_date", th.DateTimeType), + th.Property("period", th.StringType), + th.Property("project_start_date", th.DateTimeType), + ) + ), + ), th.Property("server_deleted_at", th.DateTimeType), th.Property("start_date", th.DateTimeType), th.Property("status", th.StringType), @@ -266,7 +294,7 @@ class TasksStream(TogglPaginationStream): parent_stream_type = WorkspacesStream name = "tasks" - path = "/workspaces/{workspace_id}/tasks" + path = "/api/v9/workspaces/{workspace_id}/tasks" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" records_jsonpath = "$.data[*]" @@ -290,7 +318,7 @@ class TagsStream(TogglStream): parent_stream_type = WorkspacesStream name = "tags" - path = "/workspaces/{workspace_id}/tags" + path = "/api/v9/workspaces/{workspace_id}/tags" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "at" schema = th.PropertiesList( @@ -306,52 +334,74 @@ class TagsStream(TogglStream): class TimeEntriesStream(TogglStream): """Define custom stream.""" + parent_stream_type = WorkspacesStream name = "time_entries" - path = "/me/time_entries" - primary_keys: t.ClassVar[list[str]] = ["id"] - replication_key = "at" + path = "/reports/api/v3/workspace/{workspace_id}/search/time_entries" + rest_method = "POST" + primary_keys: t.ClassVar[list[str]] = ["time_entries"] + replication_key = None schema = th.PropertiesList( - th.Property("at", th.DateTimeType), th.Property("billable", th.BooleanType), - th.Property("client_name", th.StringType), + th.Property("billable_amount_in_cents", th.IntegerType), + th.Property("currency", th.StringType), th.Property("description", th.StringType), - th.Property("duration", th.IntegerType), - th.Property("duronly", th.BooleanType), - th.Property("id", th.IntegerType), - th.Property("pid", th.IntegerType), - th.Property("project_active", th.BooleanType), - th.Property("project_color", th.StringType), + th.Property("hourly_rate_in_cents", th.IntegerType), th.Property("project_id", th.IntegerType), - th.Property("project_name", th.StringType), - th.Property("server_deleted_at", th.DateTimeType), - th.Property("start", th.DateTimeType), - th.Property("stop", th.DateTimeType), + th.Property("row_number", th.IntegerType), th.Property("tag_ids", th.ArrayType(th.IntegerType)), - th.Property("tags", th.ArrayType(th.StringType)), th.Property("task_id", th.IntegerType), - th.Property("task_name", th.StringType), - th.Property("tid", th.IntegerType), - th.Property("uid", th.IntegerType), + th.Property( + "time_entries", + th.ArrayType( + th.ObjectType( + th.Property("at", th.DateTimeType), + th.Property("id", th.IntegerType), + th.Property("seconds", th.IntegerType), + th.Property("start", th.DateTimeType), + th.Property("stop", th.DateTimeType), + ) + ), + ), th.Property("user_id", th.IntegerType), - th.Property("wid", th.IntegerType), + th.Property("username", th.StringType), th.Property("workspace_id", th.IntegerType), ).to_dict() - def get_url_params( - self, - context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, # noqa: ANN401 - ) -> dict[str, t.Any]: - """Return a dictionary of values to be used in URL parameterization. + def prepare_request_payload( + self, + context: dict | None, + next_page_token: _TToken | None, + ) -> dict | None: + """Prepare the data payload for the REST API request. Args: - context: The stream context. - next_page_token: The next page index or value. + context: Stream partition or context dictionary. + next_page_token: Token, page number or any request argument to request the + next page of data. + """ + payload = { + "start_date": self.config.get("start_date"), + "page_size": 10000, + } + + if next_page_token: + payload["first_row_number"] = int(next_page_token) + + return payload + + def get_new_paginator(self) -> BaseAPIPaginator: + """Get a fresh paginator for this API endpoint. Returns: - A dictionary of URL query parameters. + A paginator instance. """ - if self.config.get("start_date"): - return {"since": self.start_time_to_epoch(self.config.get("start_date"))} - else: - return {} + return SimpleHeaderPaginator("X-Next-Row-Number") + + def post_process( + self, + row: dict, + context: dict | None = None, # noqa: ARG002 + ) -> dict | None: + """Append or transform raw data to match expected structure.""" + row["workspace_id"] = context["workspace_id"] + return row diff --git a/tap_toggl/tap.py b/tap_toggl/tap.py index 8270f5f..7ef31a8 100644 --- a/tap_toggl/tap.py +++ b/tap_toggl/tap.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import datetime + from singer_sdk import Tap from singer_sdk import typing as th @@ -32,16 +34,9 @@ class TapToggl(Tap): "start_date", th.DateTimeType, required=False, - default="", + default=datetime.now().strftime("%Y-%m-%d"), description="The earliest record date to sync. In the format YYYY-MM-DD.", ), - th.Property( - "user_agent", - th.StringType, - required=False, - default="", - description="Inserts a user agent into the request header", - ), ).to_dict() def discover_streams(self) -> list[streams.TogglStream]: