diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 372dcf1..cecff44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,24 +16,24 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.2 + rev: 0.26.3 hooks: - id: check-dependabot - id: check-github-workflows -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.277 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 hooks: - id: ruff args: ["--fix"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v1.5.1 hooks: - id: mypy pass_filenames: true diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index dd5ce09..51b8d4d 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import typing as t from datetime import datetime, timezone from pathlib import Path @@ -55,8 +56,8 @@ def http_headers(self) -> dict: def get_next_page_token( self, response: requests.Response, - previous_token: t.Any | None, - ) -> t.Any | None: + previous_token: t.Any | None, # noqa: ANN401 + ) -> t.Any | None: # noqa: ANN401 """Return a token for identifying next page or None if no more pages.""" # If pagination is required, return a token which can be used to get the # next page. If this is the final page, return "None" to end the @@ -67,20 +68,19 @@ def get_next_page_token( elements = resp_json.get("elements") - if elements is not None: - if len(elements) == 0 or len(elements) == previous_token + 1: - return None - else: + if elements is None: page = resp_json - if len(page) == 0 or len(page) == previous_token + 1: + if len(page) in [0, previous_token + 1]: return None + elif len(elements) in [0, previous_token + 1]: + return None return previous_token + 1 def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -100,7 +100,7 @@ def get_url_params( return params - def parse_response( # noqa: PLR0912 + def parse_response( self, response: requests.Response, ) -> t.Iterable[dict]: @@ -117,72 +117,58 @@ def parse_response( # noqa: PLR0912 results = resp_json["elements"] try: columns = results[0] - except: # noqa: E722 + except Exception: # noqa: BLE001 columns = results - pass - try: - created_time = ( - columns.get("changeAuditStamps").get("created").get("time") - ) - last_modified_time = ( - columns.get("changeAuditStamps").get("lastModified").get("time") - ) - columns["created_time"] = datetime.fromtimestamp( - int(created_time) / 1000, - tz=UTC, - ).isoformat() - columns["last_modified_time"] = datetime.fromtimestamp( - int(last_modified_time) / 1000, - tz=UTC, - ).isoformat() - except: # noqa: E722, S110 - pass + with contextlib.suppress(Exception): + self._add_datetime_columns(columns) + else: results = resp_json try: columns = results - created_time = ( - columns.get("changeAuditStamps").get("created").get("time") - ) - last_modified_time = ( - columns.get("changeAuditStamps").get("lastModified").get("time") - ) - columns["created_time"] = datetime.fromtimestamp( - int(created_time) / 1000, - tz=UTC, - ).isoformat() - columns["last_modified_time"] = datetime.fromtimestamp( - int(last_modified_time) / 1000, - tz=UTC, - ).isoformat() - except: # noqa: E722 + self._add_datetime_columns(columns) + except Exception: # noqa: BLE001 columns = results - pass - - try: - account_column = columns.get("account") - account_id = int(account_column.split(":")[3]) - columns["account_id"] = account_id - except: # noqa: E722, S110 - pass - try: - campaign_column = columns.get("campaignGroup") - campaign = int(campaign_column.split(":")[3]) - columns["campaign_group_id"] = campaign - except: # noqa: E722, S110 - pass - try: + with contextlib.suppress(Exception): + self._to_id_column(columns, "account", "account_id") + + with contextlib.suppress(Exception): + self._to_id_column( + columns, + "campaignGroup", + "campaign_group_id", + ) + with contextlib.suppress(Exception): schedule_column = columns.get("runSchedule").get("start") columns["run_schedule_start"] = datetime.fromtimestamp( # noqa: DTZ006 int(schedule_column) / 1000, ).isoformat() - except: # noqa: E722, S110 - pass - - results = ( + yield from ( resp_json["elements"] if resp_json.get("elements") is not None else [columns] ) - yield from results + def _to_id_column( + self, + columns, # noqa: ANN001 + arg1, # noqa: ANN001 + arg2, # noqa: ANN001 + ) -> None: + account_column = columns.get(arg1) + account_id = int(account_column.split(":")[3]) + columns[arg2] = account_id + + def _add_datetime_columns(self, columns): # noqa: ANN202, ANN001 + created_time = columns.get("changeAuditStamps").get("created").get("time") + last_modified_time = ( + columns.get("changeAuditStamps").get("lastModified").get("time") + ) + columns["created_time"] = datetime.fromtimestamp( + int(created_time) / 1000, + tz=UTC, + ).isoformat() + columns["last_modified_time"] = datetime.fromtimestamp( + int(last_modified_time) / 1000, + tz=UTC, + ).isoformat() diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index 6bcdb0d..5e5b914 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -126,7 +126,7 @@ def url_base(self) -> str: def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -291,7 +291,7 @@ def adanalyticscolumns(self) -> list[str]: def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -330,18 +330,14 @@ def get_url_params( return params def post_process(self, row: dict, context: dict | None = None) -> dict | None: - # This function extracts day, month, and year from date rannge column + # This function extracts day, month, and year from date range column # These values are parsed with datetime function and the date is added to the day column date_range = row.get("dateRange", {}) start_date = date_range.get("start", {}) if start_date: row["day"] = datetime.strptime( - "{}-{}-{}".format( - start_date.get("year"), - start_date.get("month"), - start_date.get("day"), - ), + f'{start_date.get("year")}-{start_date.get("month")}-{start_date.get("day")}', "%Y-%m-%d", ).astimezone(UTC) @@ -361,7 +357,7 @@ class AdAnalyticsByCampaign(AdAnalyticsByCampaignInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -399,7 +395,7 @@ def get_url_params( return params def get_records(self, context: dict | None) -> t.Iterable[dict[str, t.Any]]: - """Return a dictionary of records from adAanalytics classes. + """Return a dictionary of records from adAnalytics classes. Combines request columns from multiple calls to the api, which are limited to 20 columns each. @@ -462,7 +458,7 @@ class AdAnalyticsByCampaignSecond(AdAnalyticsByCampaignInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -510,7 +506,7 @@ class AdAnalyticsByCampaignThird(AdAnalyticsByCampaignInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -603,7 +599,7 @@ class VideoAds(LinkedInAdsStream): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -628,9 +624,9 @@ def get_url_params( return params def post_process(self, row: dict, context: dict | None = None) -> dict | None: - # This function extracts day, month, and year from date rannge column + # This function extracts day, month, and year from date range column # These values are parse with datetime function and the date is added to the day column - try: + with contextlib.suppress(Exception): created_time = ( row.get("changeAuditStamps", {}).get("created", {}).get("time") ) @@ -645,9 +641,6 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None: int(last_modified_time) / 1000, tz=UTC, ).isoformat() - except: # noqa: E722, S110 - pass - return super().post_process(row, context) @property @@ -718,7 +711,7 @@ class AccountUsers(LinkedInAdsStream): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -742,15 +735,13 @@ def get_url_params( return params def post_process(self, row: dict, context: dict | None = None) -> dict | None: - # This function extracts day, month, and year from date rannge column + # This function extracts day, month, and year from date range column # These values are parsed with datetime function and the date is added to the day column - try: + with contextlib.suppress(Exception): account_user = row.get("user", {}) user = account_user.split(":")[3] row["user_person_id"] = user - except: # noqa: E722, S110 - pass - try: + with contextlib.suppress(Exception): created_time = ( row.get("changeAuditStamps", {}).get("created", {}).get("time") ) @@ -765,9 +756,6 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None: int(last_modified_time) / 1000, tz=UTC, ).isoformat() - except: # noqa: E722, S110 - pass - return super().post_process(row, context) @property @@ -852,15 +840,12 @@ class CampaignGroups(LinkedInAdsStream): @property def url_base(self) -> str: - return "https://api.linkedin.com/rest/adAccounts/{}/adCampaignGroups/{}".format( - self.config["accounts"], - self.config["campaign_group"], - ) + return f'https://api.linkedin.com/rest/adAccounts/{self.config["accounts"]}/adCampaignGroups/{self.config["campaign_group"]}' def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1147,15 +1132,12 @@ class Campaigns(LinkedInAdsStream): @property def url_base(self) -> str: - return "https://api.linkedin.com/rest/adAccounts/{}/adCampaigns/{}".format( - self.config["accounts"], - self.config["campaign"], - ) + return f'https://api.linkedin.com/rest/adAccounts/{self.config["accounts"]}/adCampaigns/{self.config["campaign"]}' def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1230,15 +1212,12 @@ class Creatives(LinkedInAdsStream): @property def url_base(self) -> str: - return "https://api.linkedin.com/rest/adAccounts/{}/creatives/urn%3Ali%3AsponsoredCreative%3A{}".format( - self.config["accounts"], - self.config["creative"], - ) + return f'https://api.linkedin.com/rest/adAccounts/{self.config["accounts"]}/creatives/urn%3Ali%3AsponsoredCreative%3A{self.config["creative"]}' def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1401,7 +1380,7 @@ def adanalyticscolumns(self) -> list[str]: def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1444,18 +1423,14 @@ def url_base(self) -> str: return "https://api.linkedin.com/rest/" def post_process(self, row: dict, context: dict | None = None) -> dict | None: - # This function extracts day, month, and year from date rannge column + # This function extracts day, month, and year from date range column # These values are parsed with datetime function and the date is added to the day column date_range = row.get("dateRange", {}) start_date = date_range.get("start", {}) if start_date: row["day"] = datetime.strptime( - "{}-{}-{}".format( - start_date.get("year"), - start_date.get("month"), - start_date.get("day"), - ), + f'{start_date.get("year")}-{start_date.get("month")}-{start_date.get("day")}', "%Y-%m-%d", ).astimezone(UTC) @@ -1475,7 +1450,7 @@ class AdAnalyticsByCreative(AdAnalyticsByCreativeInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1576,7 +1551,7 @@ class AdAnalyticsByCreativeSecond(AdAnalyticsByCreativeInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization. @@ -1624,7 +1599,7 @@ class AdAnalyticsByCreativeThird(AdAnalyticsByCreativeInit): def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: t.Any | None, + next_page_token: t.Any | None, # noqa: ANN401 ) -> dict[str, t.Any]: """Return a dictionary of values to be used in URL parameterization.