diff --git a/core/actions/declaration.ts b/core/actions/declaration.ts index bd878205f..41efa5513 100644 --- a/core/actions/declaration.ts +++ b/core/actions/declaration.ts @@ -8,26 +8,15 @@ import { ITargetableConfig } from "df/core/common"; import { Session } from "df/core/session"; -import { - actionConfigToCompiledGraphTarget, - checkExcessProperties, - strictKeysOf -} from "df/core/utils"; +import { actionConfigToCompiledGraphTarget, checkExcessProperties } from "df/core/utils"; import { dataform } from "df/protos/ts"; -/** - * Configuration options for `declaration` action types. - */ -export interface IDeclarationConfig extends IDocumentableConfig, INamedConfig, ITargetableConfig {} - -export const IDeclarationConfigProperties = strictKeysOf()([ - "columns", - "database", - "description", - "name", - "schema", - "type" -]); +interface ILegacyDeclarationConfig extends dataform.ActionConfig.DeclarationConfig { + database: string; + schema: string; + fileName: string; + type: string; +} /** * @hidden @@ -38,14 +27,16 @@ export class Declaration extends ActionBuilder { public session: Session; - constructor(session?: Session, config?: dataform.ActionConfig.DeclarationConfig) { + constructor(session?: Session, unverifiedConfig?: any) { super(session); this.session = session; - if (!config) { + if (!unverifiedConfig) { return; } + const config = this.verifyConfig(unverifiedConfig); + if (!config.name) { throw Error("Declarations must have a populated 'name' field."); } @@ -54,22 +45,15 @@ export class Declaration extends ActionBuilder { this.proto.target = this.applySessionToTarget(target, session.projectConfig); this.proto.canonicalTarget = this.applySessionToTarget(target, session.canonicalProjectConfig); - // TODO(ekrekr): load config proto column descriptors. - this.config({ description: config.description }); - } - - public config(config: IDeclarationConfig) { - checkExcessProperties( - (e: Error) => this.session.compileError(e), - config, - IDeclarationConfigProperties, - "declaration config" - ); if (config.description) { this.description(config.description); } - if (config.columns) { - this.columns(config.columns); + if (config.columns?.length) { + this.columns( + config.columns.map(columnDescriptor => + dataform.ActionConfig.ColumnDescriptor.create(columnDescriptor) + ) + ); } return this; } @@ -82,13 +66,12 @@ export class Declaration extends ActionBuilder { return this; } - public columns(columns: IColumnsDescriptor) { + public columns(columns: dataform.ActionConfig.ColumnDescriptor[]) { if (!this.proto.actionDescriptor) { this.proto.actionDescriptor = {}; } - this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray( - columns, - (e: Error) => this.session.compileError(e) + this.proto.actionDescriptor.columns = ColumnDescriptors.mapConfigProtoToCompilationProto( + columns ); return this; } @@ -114,4 +97,37 @@ export class Declaration extends ActionBuilder { VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM ); } + + private verifyConfig( + unverifiedConfig: ILegacyDeclarationConfig + ): dataform.ActionConfig.DeclarationConfig { + if (unverifiedConfig.database) { + unverifiedConfig.project = unverifiedConfig.database; + delete unverifiedConfig.database; + } + if (unverifiedConfig.schema) { + unverifiedConfig.dataset = unverifiedConfig.schema; + delete unverifiedConfig.schema; + } + if (unverifiedConfig.columns) { + // TODO(ekrekr) columns in their current config format are a difficult structure to represent + // as protos. They are nested, and use the object keys as the names. Consider a forced + // migration to the proto style column names. + unverifiedConfig.columns = ColumnDescriptors.mapLegacyObjectToConfigProto( + unverifiedConfig.columns as any + ); + } + + // TODO(ekrekr): consider moving this to a shared location after all action builders have proto + // config verifiers. + if (unverifiedConfig.type) { + delete unverifiedConfig.type; + } + + return verifyObjectMatchesProto( + dataform.ActionConfig.DeclarationConfig, + unverifiedConfig, + VerifyProtoErrorBehaviour.SHOW_DOCS_LINK + ); + } } diff --git a/core/actions/operation.ts b/core/actions/operation.ts index ad7299cd4..d1dbd11f3 100644 --- a/core/actions/operation.ts +++ b/core/actions/operation.ts @@ -1,6 +1,6 @@ import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos"; import { ActionBuilder } from "df/core/actions"; -import { ColumnDescriptors } from "df/core/column_descriptors"; +import { LegacyColumnDescriptors } from "df/core/column_descriptors"; import { Contextable, IActionConfig, @@ -213,7 +213,7 @@ export class Operation extends ActionBuilder { if (!this.proto.actionDescriptor) { this.proto.actionDescriptor = {}; } - this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray( + this.proto.actionDescriptor.columns = LegacyColumnDescriptors.mapToColumnProtoArray( columns, (e: Error) => this.session.compileError(e) ); diff --git a/core/actions/table.ts b/core/actions/table.ts index 52fe1c4f5..d9aa032b9 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -1,7 +1,7 @@ import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos"; import { ActionBuilder } from "df/core/actions"; import { Assertion } from "df/core/actions/assertion"; -import { ColumnDescriptors } from "df/core/column_descriptors"; +import { LegacyColumnDescriptors } from "df/core/column_descriptors"; import { Contextable, IActionConfig, @@ -606,7 +606,7 @@ export class Table extends ActionBuilder { if (!this.proto.actionDescriptor) { this.proto.actionDescriptor = {}; } - this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray( + this.proto.actionDescriptor.columns = LegacyColumnDescriptors.mapToColumnProtoArray( columns, (e: Error) => this.session.compileError(e) ); diff --git a/core/column_descriptors.ts b/core/column_descriptors.ts index 92227186f..f64109f14 100644 --- a/core/column_descriptors.ts +++ b/core/column_descriptors.ts @@ -6,13 +6,77 @@ import { dataform } from "df/protos/ts"; * @hidden */ export class ColumnDescriptors { + public static mapConfigProtoToCompilationProto( + columns: dataform.ActionConfig.ColumnDescriptor[] + ): dataform.IColumnDescriptor[] { + return columns.map(column => { + return dataform.ColumnDescriptor.create({ + path: column.path, + description: column.description, + tags: column.tags, + bigqueryPolicyTags: column.bigqueryPolicyTags + }); + }); + } + + public static mapLegacyObjectToConfigProto( + columns: IColumnsDescriptor + ): dataform.ActionConfig.ColumnDescriptor[] { + return Object.keys(columns) + .map(column => ColumnDescriptors.mapColumnDescriptionToProto([column], columns[column])) + .flat(); + } + + public static mapColumnDescriptionToProto( + currentPath: string[], + description: string | IRecordDescriptor + ): dataform.ActionConfig.ColumnDescriptor[] { + if (typeof description === "string") { + return [ + dataform.ColumnDescriptor.create({ + description, + path: currentPath + }) + ]; + } + const columnDescriptor: dataform.ActionConfig.ColumnDescriptor[] = !!description + ? [ + dataform.ActionConfig.ColumnDescriptor.create({ + path: currentPath, + description: description.description, + tags: typeof description.tags === "string" ? [description.tags] : description.tags, + bigqueryPolicyTags: + typeof description.bigqueryPolicyTags === "string" + ? [description.bigqueryPolicyTags] + : description.bigqueryPolicyTags + }) + ] + : []; + const nestedColumns = description.columns ? Object.keys(description.columns) : []; + return columnDescriptor.concat( + nestedColumns + .map(nestedColumn => + ColumnDescriptors.mapColumnDescriptionToProto( + currentPath.concat([nestedColumn]), + description.columns[nestedColumn] + ) + ) + .flat() + ); + } +} + +/** + * @hidden + */ +export class LegacyColumnDescriptors { public static mapToColumnProtoArray( columns: IColumnsDescriptor, reportError: (e: Error) => void ): dataform.IColumnDescriptor[] { return Object.keys(columns) .map(column => - ColumnDescriptors.mapColumnDescriptionToProto([column], columns[column], reportError) + LegacyColumnDescriptors.mapColumnDescriptionToProto([column], columns[column], reportError) ) .flat(); } @@ -54,7 +118,7 @@ export class ColumnDescriptors { return columnDescriptor.concat( nestedColumns .map(nestedColumn => - ColumnDescriptors.mapColumnDescriptionToProto( + LegacyColumnDescriptors.mapColumnDescriptionToProto( currentPath.concat([nestedColumn]), description.columns[nestedColumn], reportError diff --git a/core/main_test.ts b/core/main_test.ts index f21d0c8c0..5c85a8973 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -1345,7 +1345,6 @@ actions: columns: { nestedColumnKey: "nestedColumnVal" }, - displayName: "displayName", tags: ["tag3", "tag4"], bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], } @@ -1359,7 +1358,6 @@ actions: { bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], description: "description", - displayName: "displayName", path: ["column2Key"], tags: ["tag3", "tag4"] }, diff --git a/core/session.ts b/core/session.ts index e66b281eb..8af5eb635 100644 --- a/core/session.ts +++ b/core/session.ts @@ -161,11 +161,9 @@ export class Session { .queries(actionOptions.sqlContextable); break; case "declaration": - this.declare({ - database: sqlxConfig.database, - schema: sqlxConfig.schema, - name: sqlxConfig.name - }).config(sqlxConfig); + const declaration = new Declaration(this, sqlxConfig); + declaration.proto.fileName = utils.getCallerFile(this.rootDir); + this.actions.push(declaration); break; case "test": const testCase = this.test(sqlxConfig.name) diff --git a/protos/configs.proto b/protos/configs.proto index 4acd257ee..140c045d4 100644 --- a/protos/configs.proto +++ b/protos/configs.proto @@ -79,6 +79,9 @@ message ActionConfig { // A list of BigQuery policy tags that will be applied to the column. repeated string bigquery_policy_tags = 3; + + // A list of tags for this column which will be applied. + repeated string tags = 4; } message TableConfig {