diff --git a/core/main_test.ts b/core/main_test.ts index 02dcb2000..db46db07a 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -1676,66 +1676,67 @@ actions: nonNull: "nonNull", rowConditions: ["rowConditions1", "rowConditions2"], },`, - outputAssertions: [ - { - target: { - database: "defaultProject", - schema: "defaultDataset", - name: "dataset_name_assertions_uniqueKey_0" - }, - canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", - name: "dataset_name_assertions_uniqueKey_0" - }, - dependencyTargets: [ - { + outputAssertions: (filename: string) => + [ + { + target: { + database: "defaultProject", + schema: "defaultDataset", + name: "dataset_name_assertions_uniqueKey_0" + }, + canonicalTarget: { + database: "defaultProject", + schema: "defaultDataset", + name: "dataset_name_assertions_uniqueKey_0" + }, + dependencyTargets: [ + { + database: "project", + schema: "dataset", + name: "name" + } + ], + disabled: true, + fileName: `definitions/${filename}`, + parentAction: { database: "project", schema: "dataset", name: "name" - } - ], - disabled: true, - fileName: "definitions/filename.sqlx", - parentAction: { - database: "project", - schema: "dataset", - name: "name" - }, - query: - "\nSELECT\n *\nFROM (\n SELECT\n uniqueKey1, uniqueKey2,\n COUNT(1) AS index_row_count\n FROM `project.dataset.name`\n GROUP BY uniqueKey1, uniqueKey2\n ) AS data\nWHERE index_row_count > 1\n", - tags: ["tag1", "tag2"] - }, - { - target: { - database: "defaultProject", - schema: "defaultDataset", - name: "dataset_name_assertions_rowConditions" - }, - canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", - name: "dataset_name_assertions_rowConditions" + }, + query: + "\nSELECT\n *\nFROM (\n SELECT\n uniqueKey1, uniqueKey2,\n COUNT(1) AS index_row_count\n FROM `project.dataset.name`\n GROUP BY uniqueKey1, uniqueKey2\n ) AS data\nWHERE index_row_count > 1\n", + tags: ["tag1", "tag2"] }, - dependencyTargets: [ - { + { + target: { + database: "defaultProject", + schema: "defaultDataset", + name: "dataset_name_assertions_rowConditions" + }, + canonicalTarget: { + database: "defaultProject", + schema: "defaultDataset", + name: "dataset_name_assertions_rowConditions" + }, + dependencyTargets: [ + { + database: "project", + schema: "dataset", + name: "name" + } + ], + disabled: true, + fileName: `definitions/${filename}`, + parentAction: { database: "project", schema: "dataset", name: "name" - } - ], - disabled: true, - fileName: "definitions/filename.sqlx", - parentAction: { - database: "project", - schema: "dataset", - name: "name" - }, - query: - "\nSELECT\n 'rowConditions1' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (rowConditions1)\nUNION ALL\nSELECT\n 'rowConditions2' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (rowConditions2)\nUNION ALL\nSELECT\n 'nonNull IS NOT NULL' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (nonNull IS NOT NULL)\n", - tags: ["tag1", "tag2"] - } - ] as dataform.IAssertion[] + }, + query: + "\nSELECT\n 'rowConditions1' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (rowConditions1)\nUNION ALL\nSELECT\n 'rowConditions2' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (rowConditions2)\nUNION ALL\nSELECT\n 'nonNull IS NOT NULL' AS failing_row_condition,\n *\nFROM `project.dataset.name`\nWHERE NOT (nonNull IS NOT NULL)\n", + tags: ["tag1", "tag2"] + } + ] as dataform.IAssertion[] }; test(`for assertions`, () => { @@ -1844,18 +1845,7 @@ ${exampleActionDescriptor.inputSqlxConfigBlock} ); }); - test("for tables", () => { - const projectDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(projectDir, "workflow_settings.yaml"), - VALID_WORKFLOW_SETTINGS_YAML - ); - fs.mkdirSync(path.join(projectDir, "definitions")); - fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); - fs.writeFileSync( - path.join(projectDir, "definitions/filename.sqlx"), - ` -config { + const tableConfig = `{ type: "table", name: "name", schema: "dataset", @@ -1878,79 +1868,92 @@ ${exampleActionDescriptor.inputSqlxConfigBlock} }, ${exampleBuiltInAssertions.inputAssertionBlock} dependOnDependencyAssertions: true, - hermetic: true, -} + hermetic: true +}`; + + [ + { + filename: "table.sqlx", + fileContents: ` +config ${tableConfig} SELECT 1` - ); + }, + { + filename: "table.js", + fileContents: `publish ("name", ${tableConfig}).query(ctx => \`\n\nSELECT 1\`)` + } + ].forEach(testParameters => { + test(`for tables configured in a ${testParameters.filename} file`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + fs.writeFileSync( + path.join(projectDir, `definitions/${testParameters.filename}`), + testParameters.fileContents + ); - const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ - { - target: { - database: "project", - schema: "dataset", - name: "name" - }, - canonicalTarget: { - database: "project", - schema: "dataset", - name: "name" - }, - type: "table", - disabled: true, - hermeticity: "HERMETIC", - bigquery: { - additionalOptions: { - option1Key: "option1", - option2Key: "option2" + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + target: { + database: "project", + schema: "dataset", + name: "name" }, - clusterBy: ["clusterBy"], - labels: { - key: "val" + canonicalTarget: { + database: "project", + schema: "dataset", + name: "name" }, - partitionBy: "partitionBy", - partitionExpirationDays: 1, - requirePartitionFilter: true - }, - tags: ["tag1", "tag2"], - dependencyTargets: [ - { - database: "defaultProject", - schema: "defaultDataset", - name: "operation" - } - ], - enumType: "TABLE", - fileName: "definitions/filename.sqlx", - query: "\n\nSELECT 1", - actionDescriptor: { - ...exampleActionDescriptor.outputActionDescriptor, - // sqlxConfig.bigquery.labels are placed as bigqueryLabels. - bigqueryLabels: { - key: "val" + type: "table", + disabled: true, + hermeticity: "HERMETIC", + bigquery: { + additionalOptions: { + option1Key: "option1", + option2Key: "option2" + }, + clusterBy: ["clusterBy"], + labels: { + key: "val" + }, + partitionBy: "partitionBy", + partitionExpirationDays: 1, + requirePartitionFilter: true + }, + tags: ["tag1", "tag2"], + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset", + name: "operation" + } + ], + enumType: "TABLE", + fileName: `definitions/${testParameters.filename}`, + query: "\n\nSELECT 1", + actionDescriptor: { + ...exampleActionDescriptor.outputActionDescriptor, + // sqlxConfig.bigquery.labels are placed as bigqueryLabels. + bigqueryLabels: { + key: "val" + } } } - } - ]); - expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( - exampleBuiltInAssertions.outputAssertions - ); + ]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( + exampleBuiltInAssertions.outputAssertions(testParameters.filename) + ); + }); }); - test("for views", () => { - const projectDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(projectDir, "workflow_settings.yaml"), - VALID_WORKFLOW_SETTINGS_YAML - ); - fs.mkdirSync(path.join(projectDir, "definitions")); - fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); - fs.writeFileSync( - path.join(projectDir, "definitions/filename.sqlx"), - ` -config { + const viewConfig = `{ type: "view", name: "name", schema: "dataset", @@ -1971,75 +1974,87 @@ ${exampleActionDescriptor.inputSqlxConfigBlock} dependOnDependencyAssertions: true, hermetic: true, ${exampleBuiltInAssertions.inputAssertionBlock} -} +}`; + [ + { + filename: "view.sqlx", + fileContents: ` +config ${viewConfig} SELECT 1` - ); + }, + { + filename: "view.js", + fileContents: `publish ("name", ${viewConfig}).query(ctx => \`\n\nSELECT 1\`)` + } + ].forEach(testParameters => { + test(`for views configured in a ${testParameters.filename} file`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + fs.writeFileSync( + path.join(projectDir, `definitions/${testParameters.filename}`), + testParameters.fileContents + ); - const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ - { - target: { - database: "project", - schema: "dataset", - name: "name" - }, - canonicalTarget: { - database: "project", - schema: "dataset", - name: "name" - }, - type: "view", - disabled: true, - hermeticity: "HERMETIC", - bigquery: { - additionalOptions: { - option1Key: "option1", - option2Key: "option2" + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + target: { + database: "project", + schema: "dataset", + name: "name" }, - labels: { - key: "val" - } - }, - tags: ["tag1", "tag2"], - dependencyTargets: [ - { - database: "defaultProject", - schema: "defaultDataset", - name: "operation" - } - ], - enumType: "VIEW", - fileName: "definitions/filename.sqlx", - query: "\n\nSELECT 1", - actionDescriptor: { - ...exampleActionDescriptor.outputActionDescriptor, - // sqlxConfig.bigquery.labels are placed as bigqueryLabels. - bigqueryLabels: { - key: "val" - } - }, - materialized: true - } - ]); - expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( - exampleBuiltInAssertions.outputAssertions - ); + canonicalTarget: { + database: "project", + schema: "dataset", + name: "name" + }, + type: "view", + disabled: true, + hermeticity: "HERMETIC", + bigquery: { + additionalOptions: { + option1Key: "option1", + option2Key: "option2" + }, + labels: { + key: "val" + } + }, + tags: ["tag1", "tag2"], + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset", + name: "operation" + } + ], + enumType: "VIEW", + fileName: `definitions/${testParameters.filename}`, + query: "\n\nSELECT 1", + actionDescriptor: { + ...exampleActionDescriptor.outputActionDescriptor, + // sqlxConfig.bigquery.labels are placed as bigqueryLabels. + bigqueryLabels: { + key: "val" + } + }, + materialized: true + } + ]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( + exampleBuiltInAssertions.outputAssertions(testParameters.filename) + ); + }); }); - test("for incremental tables", () => { - const projectDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(projectDir, "workflow_settings.yaml"), - VALID_WORKFLOW_SETTINGS_YAML - ); - fs.mkdirSync(path.join(projectDir, "definitions")); - fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); - fs.writeFileSync( - path.join(projectDir, "definitions/filename.sqlx"), - ` -config { + const incrementalTableConfig = `{ type: "incremental", name: "name", schema: "dataset", @@ -2067,67 +2082,91 @@ config { ${exampleBuiltInAssertions.inputAssertionBlock} hermetic: true, } +`; + [ + { + filename: "incremental.sqlx", + fileContents: ` +config ${incrementalTableConfig} SELECT 1` - ); + }, + { + filename: "incremental.js", + fileContents: `publish ("name", ${incrementalTableConfig}).query(ctx => \`\n\n\nSELECT 1\`)` + } + ].forEach(testParameters => { + test(`for incremental tables configured in a ${testParameters.filename} file`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + fs.writeFileSync( + path.join(projectDir, `definitions/${testParameters.filename}`), + testParameters.fileContents + ); - const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ - { - target: { - database: "project", - schema: "dataset", - name: "name" - }, - canonicalTarget: { - database: "project", - schema: "dataset", - name: "name" - }, - type: "incremental", - disabled: true, - protected: false, - hermeticity: "HERMETIC", - bigquery: { - additionalOptions: { - option1Key: "option1", - option2Key: "option2" + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + target: { + database: "project", + schema: "dataset", + name: "name" }, - clusterBy: ["clusterBy"], - labels: { - key: "val" + canonicalTarget: { + database: "project", + schema: "dataset", + name: "name" }, - partitionBy: "partitionBy", - partitionExpirationDays: 1, - requirePartitionFilter: true, - updatePartitionFilter: "updatePartitionFilter" - }, - tags: ["tag1", "tag2"], - uniqueKey: ["key1", "key2"], - dependencyTargets: [ - { - database: "defaultProject", - schema: "defaultDataset", - name: "operation" - } - ], - enumType: "INCREMENTAL", - fileName: "definitions/filename.sqlx", - query: "\n\nSELECT 1", - incrementalQuery: "\n\nSELECT 1", - actionDescriptor: { - ...exampleActionDescriptor.outputActionDescriptor, - // sqlxConfig.bigquery.labels are placed as bigqueryLabels. - bigqueryLabels: { - key: "val" + type: "incremental", + disabled: true, + protected: false, + hermeticity: "HERMETIC", + bigquery: { + additionalOptions: { + option1Key: "option1", + option2Key: "option2" + }, + clusterBy: ["clusterBy"], + labels: { + key: "val" + }, + partitionBy: "partitionBy", + partitionExpirationDays: 1, + requirePartitionFilter: true, + updatePartitionFilter: "updatePartitionFilter" + }, + tags: ["tag1", "tag2"], + uniqueKey: ["key1", "key2"], + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset", + name: "operation" + } + ], + enumType: "INCREMENTAL", + fileName: `definitions/${testParameters.filename}`, + query: "\n\n\nSELECT 1", + incrementalQuery: "\n\n\nSELECT 1", + actionDescriptor: { + ...exampleActionDescriptor.outputActionDescriptor, + // sqlxConfig.bigquery.labels are placed as bigqueryLabels. + bigqueryLabels: { + key: "val" + } } } - } - ]); - expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( - exampleBuiltInAssertions.outputAssertions - ); + ]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( + exampleBuiltInAssertions.outputAssertions(testParameters.filename) + ); + }); }); test(`for operations`, () => {