Declaration configs as protos (#1767)

core/actions/declaration.ts

@@ -8,26 +8,15 @@ import {
   ITargetableConfig
 } from "df/core/common";
 import { Session } from "df/core/session";
-import {
-  actionConfigToCompiledGraphTarget,
-  checkExcessProperties,
-  strictKeysOf
-} from "df/core/utils";
+import { actionConfigToCompiledGraphTarget, checkExcessProperties } from "df/core/utils";
 import { dataform } from "df/protos/ts";
 
-/**
- * Configuration options for `declaration` action types.
- */
-export interface IDeclarationConfig extends IDocumentableConfig, INamedConfig, ITargetableConfig {}
-
-export const IDeclarationConfigProperties = strictKeysOf<IDeclarationConfig>()([
-  "columns",
-  "database",
-  "description",
-  "name",
-  "schema",
-  "type"
-]);
+interface ILegacyDeclarationConfig extends dataform.ActionConfig.DeclarationConfig {
+  database: string;
+  schema: string;
+  fileName: string;
+  type: string;
+}
 
 /**
  * @hidden
@@ -38,14 +27,16 @@ export class Declaration extends ActionBuilder<dataform.Declaration> {
 
   public session: Session;
 
-  constructor(session?: Session, config?: dataform.ActionConfig.DeclarationConfig) {
+  constructor(session?: Session, unverifiedConfig?: any) {
     super(session);
     this.session = session;
 
-    if (!config) {
+    if (!unverifiedConfig) {
       return;
     }
 
+    const config = this.verifyConfig(unverifiedConfig);
+
     if (!config.name) {
       throw Error("Declarations must have a populated 'name' field.");
     }
@@ -54,22 +45,15 @@ export class Declaration extends ActionBuilder<dataform.Declaration> {
     this.proto.target = this.applySessionToTarget(target, session.projectConfig);
     this.proto.canonicalTarget = this.applySessionToTarget(target, session.canonicalProjectConfig);
 
-    // TODO(ekrekr): load config proto column descriptors.
-    this.config({ description: config.description });
-  }
-
-  public config(config: IDeclarationConfig) {
-    checkExcessProperties(
-      (e: Error) => this.session.compileError(e),
-      config,
-      IDeclarationConfigProperties,
-      "declaration config"
-    );
     if (config.description) {
       this.description(config.description);
     }
-    if (config.columns) {
-      this.columns(config.columns);
+    if (config.columns?.length) {
+      this.columns(
+        config.columns.map(columnDescriptor =>
+          dataform.ActionConfig.ColumnDescriptor.create(columnDescriptor)
+        )
+      );
     }
     return this;
   }
@@ -82,13 +66,12 @@ export class Declaration extends ActionBuilder<dataform.Declaration> {
     return this;
   }
 
-  public columns(columns: IColumnsDescriptor) {
+  public columns(columns: dataform.ActionConfig.ColumnDescriptor[]) {
     if (!this.proto.actionDescriptor) {
       this.proto.actionDescriptor = {};
     }
-    this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray(
-      columns,
-      (e: Error) => this.session.compileError(e)
+    this.proto.actionDescriptor.columns = ColumnDescriptors.mapConfigProtoToCompilationProto(
+      columns
     );
     return this;
   }
@@ -114,4 +97,37 @@ export class Declaration extends ActionBuilder<dataform.Declaration> {
       VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM
     );
   }
+
+  private verifyConfig(
+    unverifiedConfig: ILegacyDeclarationConfig
+  ): dataform.ActionConfig.DeclarationConfig {
+    if (unverifiedConfig.database) {
+      unverifiedConfig.project = unverifiedConfig.database;
+      delete unverifiedConfig.database;
+    }
+    if (unverifiedConfig.schema) {
+      unverifiedConfig.dataset = unverifiedConfig.schema;
+      delete unverifiedConfig.schema;
+    }
+    if (unverifiedConfig.columns) {
+      // TODO(ekrekr) columns in their current config format are a difficult structure to represent
+      // as protos. They are nested, and use the object keys as the names. Consider a forced
+      // migration to the proto style column names.
+      unverifiedConfig.columns = ColumnDescriptors.mapLegacyObjectToConfigProto(
+        unverifiedConfig.columns as any
+      );
+    }
+
+    // TODO(ekrekr): consider moving this to a shared location after all action builders have proto
+    // config verifiers.
+    if (unverifiedConfig.type) {
+      delete unverifiedConfig.type;
+    }
+
+    return verifyObjectMatchesProto(
+      dataform.ActionConfig.DeclarationConfig,
+      unverifiedConfig,
+      VerifyProtoErrorBehaviour.SHOW_DOCS_LINK
+    );
+  }
 }

core/actions/operation.ts

@@ -1,6 +1,6 @@
 import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
 import { ActionBuilder } from "df/core/actions";
-import { ColumnDescriptors } from "df/core/column_descriptors";
+import { LegacyColumnDescriptors } from "df/core/column_descriptors";
 import {
   Contextable,
   IActionConfig,
@@ -213,7 +213,7 @@ export class Operation extends ActionBuilder<dataform.Operation> {
     if (!this.proto.actionDescriptor) {
       this.proto.actionDescriptor = {};
     }
-    this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray(
+    this.proto.actionDescriptor.columns = LegacyColumnDescriptors.mapToColumnProtoArray(
       columns,
       (e: Error) => this.session.compileError(e)
     );

core/actions/table.ts

@@ -1,7 +1,7 @@
 import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
 import { ActionBuilder } from "df/core/actions";
 import { Assertion } from "df/core/actions/assertion";
-import { ColumnDescriptors } from "df/core/column_descriptors";
+import { LegacyColumnDescriptors } from "df/core/column_descriptors";
 import {
   Contextable,
   IActionConfig,
@@ -606,7 +606,7 @@ export class Table extends ActionBuilder<dataform.Table> {
     if (!this.proto.actionDescriptor) {
       this.proto.actionDescriptor = {};
     }
-    this.proto.actionDescriptor.columns = ColumnDescriptors.mapToColumnProtoArray(
+    this.proto.actionDescriptor.columns = LegacyColumnDescriptors.mapToColumnProtoArray(
       columns,
       (e: Error) => this.session.compileError(e)
     );

core/column_descriptors.ts

@@ -6,13 +6,77 @@ import { dataform } from "df/protos/ts";
  * @hidden
  */
 export class ColumnDescriptors {
+  public static mapConfigProtoToCompilationProto(
+    columns: dataform.ActionConfig.ColumnDescriptor[]
+  ): dataform.IColumnDescriptor[] {
+    return columns.map(column => {
+      return dataform.ColumnDescriptor.create({
+        path: column.path,
+        description: column.description,
+        tags: column.tags,
+        bigqueryPolicyTags: column.bigqueryPolicyTags
+      });
+    });
+  }
+
+  public static mapLegacyObjectToConfigProto(
+    columns: IColumnsDescriptor
+  ): dataform.ActionConfig.ColumnDescriptor[] {
+    return Object.keys(columns)
+      .map(column => ColumnDescriptors.mapColumnDescriptionToProto([column], columns[column]))
+      .flat();
+  }
+
+  public static mapColumnDescriptionToProto(
+    currentPath: string[],
+    description: string | IRecordDescriptor
+  ): dataform.ActionConfig.ColumnDescriptor[] {
+    if (typeof description === "string") {
+      return [
+        dataform.ColumnDescriptor.create({
+          description,
+          path: currentPath
+        })
+      ];
+    }
+    const columnDescriptor: dataform.ActionConfig.ColumnDescriptor[] = !!description
+      ? [
+          dataform.ActionConfig.ColumnDescriptor.create({
+            path: currentPath,
+            description: description.description,
+            tags: typeof description.tags === "string" ? [description.tags] : description.tags,
+            bigqueryPolicyTags:
+              typeof description.bigqueryPolicyTags === "string"
+                ? [description.bigqueryPolicyTags]
+                : description.bigqueryPolicyTags
+          })
+        ]
+      : [];
+    const nestedColumns = description.columns ? Object.keys(description.columns) : [];
+    return columnDescriptor.concat(
+      nestedColumns
+        .map(nestedColumn =>
+          ColumnDescriptors.mapColumnDescriptionToProto(
+            currentPath.concat([nestedColumn]),
+            description.columns[nestedColumn]
+          )
+        )
+        .flat()
+    );
+  }
+}
+
+/**
+ * @hidden
+ */
+export class LegacyColumnDescriptors {
   public static mapToColumnProtoArray(
     columns: IColumnsDescriptor,
     reportError: (e: Error) => void
   ): dataform.IColumnDescriptor[] {
     return Object.keys(columns)
       .map(column =>
-        ColumnDescriptors.mapColumnDescriptionToProto([column], columns[column], reportError)
+        LegacyColumnDescriptors.mapColumnDescriptionToProto([column], columns[column], reportError)
       )
       .flat();
   }
@@ -54,7 +118,7 @@ export class ColumnDescriptors {
     return columnDescriptor.concat(
       nestedColumns
         .map(nestedColumn =>
-          ColumnDescriptors.mapColumnDescriptionToProto(
+          LegacyColumnDescriptors.mapColumnDescriptionToProto(
             currentPath.concat([nestedColumn]),
             description.columns[nestedColumn],
             reportError

core/main_test.ts

@@ -1345,7 +1345,6 @@ actions:
       columns: {
         nestedColumnKey: "nestedColumnVal"
       },
-      displayName: "displayName",
       tags: ["tag3", "tag4"],
       bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"],
     }
@@ -1359,7 +1358,6 @@ actions:
           {
             bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"],
             description: "description",
-            displayName: "displayName",
             path: ["column2Key"],
             tags: ["tag3", "tag4"]
           },

core/session.ts

@@ -161,11 +161,9 @@ export class Session {
           .queries(actionOptions.sqlContextable);
         break;
       case "declaration":
-        this.declare({
-          database: sqlxConfig.database,
-          schema: sqlxConfig.schema,
-          name: sqlxConfig.name
-        }).config(sqlxConfig);
+        const declaration = new Declaration(this, sqlxConfig);
+        declaration.proto.fileName = utils.getCallerFile(this.rootDir);
+        this.actions.push(declaration);
         break;
       case "test":
         const testCase = this.test(sqlxConfig.name)

protos/configs.proto

@@ -79,6 +79,9 @@ message ActionConfig {
 
     // A list of BigQuery policy tags that will be applied to the column.
     repeated string bigquery_policy_tags = 3;
+
+    // A list of tags for this column which will be applied.
+    repeated string tags = 4;
   }
 
   message TableConfig {