Fix REST fields reported as not found and reveal more info in error log

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.4.4
+  * Remove fields from REST that the API reports as not found [#78](https://github.com/singer-io/tap-zuora/pull/78)
+
 ## 1.4.3
   * Fix Dependabot issue [#77](https://github.com/singer-io/tap-zuora/pull/77)
 

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="tap-zuora",
-    version="1.4.3",
+    version="1.4.4",
     description="Singer.io tap for extracting data from the Zuora API",
     author="Stitch",
     url="https://singer.io",

tap_zuora/client.py

@@ -10,6 +10,7 @@
     BadCredentialsException,
     RateLimitException,
     RetryableException,
+    InvalidValueException,
 )
 from tap_zuora.utils import make_aqua_payload
 
@@ -34,6 +35,16 @@
 
 LOGGER = singer.get_logger()
 
+def is_invalid_value_response(resp):
+    "Check for known structure of invalid value 400 response."
+    try:
+        errors = resp.json()['Errors']
+        for e in errors:
+            if e['Code'] == "INVALID_VALUE":
+                return True
+    except:
+        pass
+    return False
 
 class Client:  # pylint: disable=too-many-instance-attributes
     def __init__(

tap_zuora/discover.py

@@ -25,24 +25,78 @@
 ]
 
 UNSUPPORTED_FIELDS_FOR_REST = {
-    "Account": ["SequenceSetId"],
+    "Account": ["SequenceSetId",
+                "LastMetricsUpdate",
+                "PartnerAccount",
+                "PaymentMethodCascadingConsent",
+                "RollUpUsage",
+                "PaymentMethodPriorityId",
+    ],
     "Amendment": [
         "BookingDate",
         "EffectivePolicy",
         "NewRatePlanId",
         "RemovedRatePlanId",
         "SubType",
     ],
-    "BillingRun": ["BillingRunType", "NumberOfCreditMemos", "PostedDate"],
-    "Export": ["Encoding"],
-    "Invoice": ["PaymentTerm", "SourceType", "TaxMessage", "TaxStatus", "TemplateId"],
-    "InvoiceItem": ["Balance", "ExcludeItemBillingFromRevenueAccounting"],
+    "BillingRun": ["BillingRunType", "NumberOfCreditMemos", "PostedDate", "Name"],
+    "Contact": ["AsBillTo", "AsShipTo", "AsSoldTo"],
+    "Export": ["Encoding", "SnowflakeWarehouse", "SourceData", "WarehouseSize"],
+    "Invoice": ["PaymentTerm",
+                "SourceType",
+                "TaxMessage",
+                "TaxStatus",
+                "TemplateId",
+                "CreditMemoAmount",
+                "Currency",
+                "EInvoiceErrorCode",
+                "EInvoiceErrorMessage",
+                "EInvoiceFileId",
+                "EInvoiceStatus",
+                "InvoiceGroupNumber",
+                "PaymentLink",
+                "SequenceSetId",
+    ],
+    "InvoiceItem": ["Balance",
+                    "ExcludeItemBillingFromRevenueAccounting",
+                    "ChargeNumber",
+                    "NumberOfDeliveries",
+                    "PurchaseOrderNumber",
+                    "ReflectDiscountInNetAmount",
+                    "SubscriptionNumber",
+    ],
     "InvoiceItemAdjustment": ["ExcludeItemBillingFromRevenueAccounting"],
-    "PaymentMethod": ["StoredCredentialProfileId"],
+    "OrderAction": ["ClearingExistingShipToContact"],
+    "OrderLineItem": ["ShipTo"],
+    "PaymentMethod": ["StoredCredentialProfileId", "PaymentMethodTokenId"],
+    "Product": ["ProductNumber"],
     "ProductRatePlanCharge": [
         "ExcludeItemBillingFromRevenueAccounting",
         "ExcludeItemBookingFromRevenueAccounting",
+        "ApplyToBillingPeriodPartially",
+        "CommitmentType",
+        "Formula",
+        "IsAllocationEligible",
+        "IsCommitted",
+        "IsRollover",
+        "IsStackedDiscount",
+        "IsUnbilled",
+        "PriceUpsellQuantityStacked",
+        "ProductCategory",
+        "ProductClass",
+        "ProductFamily",
+        "ProductLine",
+        "ProductRatePlanChargeNumber",
+        "ProrationOption",
+        "RatingGroupsOperatorType",
+        "ReflectDiscountInNetAmount",
+        "RevenueAmortizationMethod",
+        "RevenueRecognitionTiming",
+        "RolloverApply",
+        "RolloverPeriods",
+        "SpecificListPriceBase",
     ],
+    "RatePlan": ["Reverted"],
     "RatePlanCharge": [
         "AmendedByOrderOn",
         "CreditOption",
@@ -58,8 +112,33 @@
         "PrepaidTotalQuantity",
         "PrepaidUom",
         "ValidityPeriodType",
+        "ApplyToBillingPeriodPartially",
+        "ChargeFunction",
+        "CommitmentLevel",
+        "CommitmentType",
+        "IsCommitted",
+        "IsRollover",
+        "PriceUpsellQuantityStacked",
+        "RatingGroupsOperatorType",
+        "ReflectDiscountInNetAmount",
+        "RevenueAmortizationMethod",
+        "RevenueRecognitionTiming",
+        "Reverted",
+        "RolloverApply",
+        "RolloverPeriodLength",
+        "RolloverPeriods",
+        "SpecificListPriceBase",
+    ],
+    "RevenueRecognitionEventsTransaction": ["UsageChargeName", "UsageChargeNumber"],
+    "Subscription": ["IsLatestVersion",
+                     "LastBookingDate",
+                     "PaymentTerm",
+                     "Revision",
+                     "Currency",
+                     "InvoiceGroupNumber",
+                     "InvoiceTemplateId",
+                     "SequenceSetId",
     ],
-    "Subscription": ["IsLatestVersion", "LastBookingDate", "PaymentTerm", "Revision"],
     "TaxationItem": ["Balance", "CreditAmount", "PaymentAmount"],
     "Usage": ["ImportId"],
 }

tap_zuora/exceptions.py

@@ -7,8 +7,14 @@ def __init__(self, resp):
 class ApiException(Exception):
     def __init__(self, resp):
         self.resp = resp
-        super().__init__("{0.status_code}: {0.content}".format(self.resp))
+        super().__init__(f"{resp.status_code}: {resp.content}")
 
+class InvalidValueException(Exception):
+    def __init__(self, resp, stream_name):
+        self.resp = resp
+        self.stream_name = stream_name
+        invalid_value_errors = [e for e in resp.json()['Errors'] if e["Code"] == "INVALID_VALUE"]
+        super().__init__(f"{stream_name} - Invalid Values in Request ({resp.status_code}), Errors: {invalid_value_errors}")
 
 class RetryableException(ApiException):
     """Class to mark an ApiException as retryable."""

tap_zuora/sync.py

@@ -6,10 +6,11 @@
 import pendulum
 import singer
 from singer import transform
+from requests.exceptions import HTTPError
 
 from tap_zuora import apis
 from tap_zuora.client import Client
-from tap_zuora.exceptions import ApiException, FileIdNotFoundException
+from tap_zuora.exceptions import ApiException, FileIdNotFoundException, InvalidValueException
 
 PARTNER_ID = "salesforce"
 DEFAULT_POLL_INTERVAL = 60
@@ -245,15 +246,18 @@ def sync_rest_stream(client: Client, state: Dict, stream: Dict, counter):
         sync_started = pendulum.utcnow()
         start_date = state["bookmarks"][stream["tap_stream_id"]][stream["replication_key"]]
         start_pen = pendulum.parse(start_date)
-        counter = iterate_rest_query_window(
-            client,
-            state,
-            stream,
-            counter,
-            start_pen,
-            sync_started,
-            window_length_in_seconds,
-        )
+        try:
+            counter = iterate_rest_query_window(
+                client,
+                state,
+                stream,
+                counter,
+                start_pen,
+                sync_started,
+                window_length_in_seconds,
+            )
+        except HTTPError as ex:
+            raise InvalidValueException(ex.response, stream_name=stream["tap_stream_id"]) from ex
     else:
         job_id = apis.Rest.create_job(client, stream)
         file_ids = poll_job_until_done(job_id, client, apis.Rest)

tests/test_zuora_all_fields.py

@@ -33,7 +33,7 @@ def name(self):
     def test_run(self):
         """Executing tap-tester scenarios for both types of zuora APIs AQUA and
         REST."""
-        # Testing for only AQUA mode to reduce the execution time
+        self.run_test("REST")
         self.run_test("AQUA")
 
     def run_test(self, api_type):
@@ -45,10 +45,10 @@ def run_test(self, api_type):
         self.zuora_api_type = api_type
 
         # Streams to verify all fields tests
-        expected_streams = {"Account"}  
-        self.assertNotEqual(JIRA_CLIENT.get_status_category('TDL-26953'), 
-                    'done',  
-                    msg='JIRA ticket has moved to done, re-add RefundTransactionLog stream to testable streams')  
+        expected_streams = {"Account"}
+        self.assertNotEqual(JIRA_CLIENT.get_status_category('TDL-26953'),
+                    'done',
+                    msg='JIRA ticket has moved to done, re-add RefundTransactionLog stream to testable streams')
 
         expected_automatic_fields = self.expected_automatic_fields()
         conn_id = connections.ensure_connection(self, original_properties=False)
@@ -69,7 +69,7 @@ def run_test(self, api_type):
             stream_id, stream_name = catalog["stream_id"], catalog["stream_name"]
             catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id)
             fields_from_field_level_md = [
-                md_entry["breadcrumb"][1] for md_entry in catalog_entry["metadata"] if md_entry["breadcrumb"] != []
+                md_entry["breadcrumb"][1] for md_entry in catalog_entry["metadata"] if md_entry["breadcrumb"] != [] and md_entry['metadata']['inclusion'] != 'unsupported'
             ]
             stream_to_all_catalog_fields[stream_name] = set(fields_from_field_level_md)