diff --git a/mastering-carol/sql-pipelines/pipedemo.csql b/mastering-carol/sql-pipelines/pipedemo.csql new file mode 100644 index 0000000..d843efa --- /dev/null +++ b/mastering-carol/sql-pipelines/pipedemo.csql @@ -0,0 +1,35 @@ +WITH organization_apinvoice AS ( + SELECT + null AS _orgid, + null AS businesspartner_id, + null AS docreference, + null AS invoice_id, + null AS docissue, + null AS erp_id, + null AS protheus_id, + null AS transactiontype, + null AS deleted, + null AS installmentqty + --metadata-- + FROM ( + SELECT * EXCEPT(ranking) + FROM ( + SELECT ROW_NUMBER() OVER (partition BY mdmId ORDER BY mdmCounterForEntity DESC) ranking, * + FROM stg_protheus_carol_organization + --timestamp-- WHERE mdmCounterForEntity__DATETIME__ > SAFE.DATETIME(TIMESTAMP_MICROS(SAFE_CAST({{start_from}} AS INT64))) + ) + WHERE ranking = 1 + ) AS stg +), +combinedSources AS ( + SELECT * FROM organization_apinvoice +), +processedData AS ( + SELECT * + FROM combinedSources + -- WHERE + -- rejection rules +) + +SELECT * FROM processedData + diff --git a/mastering-carol/sql-pipelines/pipelines.json b/mastering-carol/sql-pipelines/pipelines.json index 64b9b5c..98fd6ca 100644 --- a/mastering-carol/sql-pipelines/pipelines.json +++ b/mastering-carol/sql-pipelines/pipelines.json @@ -27,6 +27,29 @@ ] } }, + { + "useBatchNotification": true, + "pipelineName": "pipedemo", + "pipelineDescription": "Pipeline durante demo Carol", + "outputDataModelName": "organization", + "overlapDeltaMinutes": 60, + "saveToCds": false, + "saveToUnified": true, + "sendToSubscriptions": true, + "cronExpressions": ["0 0 * * * ?"], + "timeZone": "America/Sao_Paulo", + "prepareScripts": [""], + "processScript": "pipedemo.csql", + "sourceEntities": { + "dataModels": [], + "stagings": [ + { + "connectorName": "protheus_carol", + "stagingName": "sa2" + } + ] + } + }, { "useBatchNotification": true, "pipelineName": "organization_pipeline",