Add stream_name to invalid_value eception, fix current issues

singer-io · Feb 4, 2025 · 855bca4 · 855bca4
1 parent 9729e7c
commit 855bca4
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 21 deletions.
diff --git a/tap_zuora/client.py b/tap_zuora/client.py
@@ -170,8 +170,6 @@ def _retryable_request(self, method: str, url: str, stream=False, url_check=Fals
         # 502(Bad Gateway), 503(service unavailable), 504(Gateway Timeout)
         if resp.status_code in [500, 502, 503, 504]:
             raise RetryableException(resp)
-        if resp.status_code == 400 and is_invalid_value_response(resp):
-            raise InvalidValueException(resp)
         self.check_for_error(resp, url_check)
         return resp
 

diff --git a/tap_zuora/discover.py b/tap_zuora/discover.py
@@ -25,24 +25,78 @@
 ]
 
 UNSUPPORTED_FIELDS_FOR_REST = {
-    "Account": ["SequenceSetId"],
+    "Account": ["SequenceSetId",
+                "LastMetricsUpdate",
+                "PartnerAccount",
+                "PaymentMethodCascadingConsent",
+                "RollUpUsage",
+                "PaymentMethodPriorityId",
+    ],
     "Amendment": [
         "BookingDate",
         "EffectivePolicy",
         "NewRatePlanId",
         "RemovedRatePlanId",
         "SubType",
     ],
-    "BillingRun": ["BillingRunType", "NumberOfCreditMemos", "PostedDate"],
-    "Export": ["Encoding"],
-    "Invoice": ["PaymentTerm", "SourceType", "TaxMessage", "TaxStatus", "TemplateId"],
-    "InvoiceItem": ["Balance", "ExcludeItemBillingFromRevenueAccounting"],
+    "BillingRun": ["BillingRunType", "NumberOfCreditMemos", "PostedDate", "Name"],
+    "Contact": ["AsBillTo", "AsShipTo", "AsSoldTo"],
+    "Export": ["Encoding", "SnowflakeWarehouse", "SourceData", "WarehouseSize"],
+    "Invoice": ["PaymentTerm",
+                "SourceType",
+                "TaxMessage",
+                "TaxStatus",
+                "TemplateId",
+                "CreditMemoAmount",
+                "Currency",
+                "EInvoiceErrorCode",
+                "EInvoiceErrorMessage",
+                "EInvoiceFileId",
+                "EInvoiceStatus",
+                "InvoiceGroupNumber",
+                "PaymentLink",
+                "SequenceSetId",
+    ],
+    "InvoiceItem": ["Balance",
+                    "ExcludeItemBillingFromRevenueAccounting",
+                    "ChargeNumber",
+                    "NumberOfDeliveries",
+                    "PurchaseOrderNumber",
+                    "ReflectDiscountInNetAmount",
+                    "SubscriptionNumber",
+    ],
     "InvoiceItemAdjustment": ["ExcludeItemBillingFromRevenueAccounting"],
-    "PaymentMethod": ["StoredCredentialProfileId"],
+    "OrderAction": ["ClearingExistingShipToContact"],
+    "OrderLineItem": ["ShipTo"],
+    "PaymentMethod": ["StoredCredentialProfileId", "PaymentMethodTokenId"],
+    "Product": ["ProductNumber"],
     "ProductRatePlanCharge": [
         "ExcludeItemBillingFromRevenueAccounting",
         "ExcludeItemBookingFromRevenueAccounting",
+        "ApplyToBillingPeriodPartially",
+        "CommitmentType",
+        "Formula",
+        "IsAllocationEligible",
+        "IsCommitted",
+        "IsRollover",
+        "IsStackedDiscount",
+        "IsUnbilled",
+        "PriceUpsellQuantityStacked",
+        "ProductCategory",
+        "ProductClass",
+        "ProductFamily",
+        "ProductLine",
+        "ProductRatePlanChargeNumber",
+        "ProrationOption",
+        "RatingGroupsOperatorType",
+        "ReflectDiscountInNetAmount",
+        "RevenueAmortizationMethod",
+        "RevenueRecognitionTiming",
+        "RolloverApply",
+        "RolloverPeriods",
+        "SpecificListPriceBase",
     ],
+    "RatePlan": ["Reverted"],
     "RatePlanCharge": [
         "AmendedByOrderOn",
         "CreditOption",
@@ -58,8 +112,33 @@
         "PrepaidTotalQuantity",
         "PrepaidUom",
         "ValidityPeriodType",
+        "ApplyToBillingPeriodPartially",
+        "ChargeFunction",
+        "CommitmentLevel",
+        "CommitmentType",
+        "IsCommitted",
+        "IsRollover",
+        "PriceUpsellQuantityStacked",
+        "RatingGroupsOperatorType",
+        "ReflectDiscountInNetAmount",
+        "RevenueAmortizationMethod",
+        "RevenueRecognitionTiming",
+        "Reverted",
+        "RolloverApply",
+        "RolloverPeriodLength",
+        "RolloverPeriods",
+        "SpecificListPriceBase",
+    ],
+    "RevenueRecognitionEventsTransaction": ["UsageChargeName", "UsageChargeNumber"],
+    "Subscription": ["IsLatestVersion",
+                     "LastBookingDate",
+                     "PaymentTerm",
+                     "Revision",
+                     "Currency",
+                     "InvoiceGroupNumber",
+                     "InvoiceTemplateId",
+                     "SequenceSetId",
     ],
-    "Subscription": ["IsLatestVersion", "LastBookingDate", "PaymentTerm", "Revision"],
     "TaxationItem": ["Balance", "CreditAmount", "PaymentAmount"],
     "Usage": ["ImportId"],
 }

diff --git a/tap_zuora/exceptions.py b/tap_zuora/exceptions.py
@@ -10,10 +10,11 @@ def __init__(self, resp):
         super().__init__(f"{resp.status_code}: {resp.content}")
 
 class InvalidValueException(Exception):
-    def __init__(self, resp):
+    def __init__(self, resp, stream_name):
         self.resp = resp
+        self.stream_name = stream_name
         invalid_value_errors = [e for e in resp.json()['Errors'] if e["Code"] == "INVALID_VALUE"]
-        super().__init__(f"Invalid Values in Request ({resp.status_code}), Errors: {invalid_value_errors}")
+        super().__init__(f"{stream_name} - Invalid Values in Request ({resp.status_code}), Errors: {invalid_value_errors}")
 
 class RetryableException(ApiException):
     """Class to mark an ApiException as retryable."""

diff --git a/tap_zuora/sync.py b/tap_zuora/sync.py
@@ -6,10 +6,11 @@
 import pendulum
 import singer
 from singer import transform
+from requests.exceptions import HTTPError
 
 from tap_zuora import apis
 from tap_zuora.client import Client
-from tap_zuora.exceptions import ApiException, FileIdNotFoundException
+from tap_zuora.exceptions import ApiException, FileIdNotFoundException, InvalidValueException
 
 PARTNER_ID = "salesforce"
 DEFAULT_POLL_INTERVAL = 60
@@ -245,15 +246,18 @@ def sync_rest_stream(client: Client, state: Dict, stream: Dict, counter):
         sync_started = pendulum.utcnow()
         start_date = state["bookmarks"][stream["tap_stream_id"]][stream["replication_key"]]
         start_pen = pendulum.parse(start_date)
-        counter = iterate_rest_query_window(
-            client,
-            state,
-            stream,
-            counter,
-            start_pen,
-            sync_started,
-            window_length_in_seconds,
-        )
+        try:
+            counter = iterate_rest_query_window(
+                client,
+                state,
+                stream,
+                counter,
+                start_pen,
+                sync_started,
+                window_length_in_seconds,
+            )
+        except HTTPError as ex:
+            raise InvalidValueException(ex.response, stream_name=stream["tap_stream_id"]) from ex
     else:
         job_id = apis.Rest.create_job(client, stream)
         file_ids = poll_job_until_done(job_id, client, apis.Rest)