Skip to content

Commit

Permalink
Make ref() work for tables in foreign schemas. (#279)
Browse files Browse the repository at this point in the history
* Make ref() work when referencing tables in foreign schemas.

* cleanuo

* updated package version
BenBirt authored and probot-auto-merge[bot] committed Jun 26, 2019
1 parent 3707b08 commit 1f4daea
Showing 10 changed files with 36 additions and 101 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ bazel-*

.idea/
.vscode/
.history/

bazel-*

17 changes: 0 additions & 17 deletions core/assertion.ts
Original file line number Diff line number Diff line change
@@ -38,23 +38,6 @@ export class Assertion {
}
}

export class SqlxAssertionContext {
private assertion?: Assertion;

constructor(assertion: Assertion) {
this.assertion = assertion;
}

public ref(name: string) {
this.assertion.dependencies(name);
return this.resolve(name);
}

public resolve(name: string) {
return this.assertion.session.resolve(name);
}
}

export class AssertionContext {
private assertion?: Assertion;

18 changes: 6 additions & 12 deletions core/compilers.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { AssertionContext, SqlxAssertionContext } from "@dataform/core/assertion";
import { OperationContext, SqlxOperationContext } from "@dataform/core/operation";
import { AssertionContext } from "@dataform/core/assertion";
import { OperationContext } from "@dataform/core/operation";
import { ISqlxParseResults, parseSqlx } from "@dataform/core/sqlx_parser";
import { SqlxTableContext, TableContext } from "@dataform/core/table";
import { TableContext } from "@dataform/core/table";
import * as utils from "@dataform/core/utils";

export function compile(code: string, path: string) {
@@ -115,9 +115,7 @@ switch (sqlxConfig.type) {
case "incremental":
case "inline": {
action.query(ctx => {
${getFunctionPropertyNames(SqlxTableContext.prototype)
.map(name => `const ${name} = ctx.${name}.bind(ctx);`)
.join("\n")}
${["self", "ref", "resolve"].map(name => `const ${name} = ctx.${name}.bind(ctx);`).join("\n")}
${results.js}
if (hasIncremental) {
action.where(\`${results.incremental}\`);
@@ -136,19 +134,15 @@ switch (sqlxConfig.type) {
}
case "assertion": {
action.query(ctx => {
${getFunctionPropertyNames(SqlxAssertionContext.prototype)
.map(name => `const ${name} = ctx.${name}.bind(ctx);`)
.join("\n")}
${["ref", "resolve"].map(name => `const ${name} = ctx.${name}.bind(ctx);`).join("\n")}
${results.js}
return \`${results.sql[0]}\`;
});
break;
}
case "operations": {
action.queries(ctx => {
${getFunctionPropertyNames(SqlxOperationContext.prototype)
.map(name => `const ${name} = ctx.${name}.bind(ctx);`)
.join("\n")}
${["self", "ref", "resolve"].map(name => `const ${name} = ctx.${name}.bind(ctx);`).join("\n")}
${results.js}
const operations = [${results.sql.map(sql => `\`${sql}\``)}];
return operations;
21 changes: 0 additions & 21 deletions core/operation.ts
Original file line number Diff line number Diff line change
@@ -43,27 +43,6 @@ export class Operation {
}
}

export class SqlxOperationContext {
private operation?: Operation;

constructor(operation: Operation) {
this.operation = operation;
}

public self(): string {
return this.resolve(this.operation.proto.name);
}

public ref(name: string) {
this.operation.dependencies(name);
return this.resolve(name);
}

public resolve(name: string) {
return this.operation.session.resolve(name);
}
}

export class OperationContext {
private operation?: Operation;

25 changes: 15 additions & 10 deletions core/session.ts
Original file line number Diff line number Diff line change
@@ -144,28 +144,33 @@ export class Session {
public target(target: string, defaultSchema?: string): dataform.ITarget {
const suffix = !!this.config.schemaSuffix ? `_${this.config.schemaSuffix}` : "";

// TODO: This should probably throw (perhaps in a future breaking version of @dataform/core).
// It should be impossible that this codepath gets hit (unless potentially somebody is calling this function using the JS API?).
if (target.includes(".")) {
const schema = target.split(".")[0];
const name = target.split(".")[1];
const [schema, name] = target.split(".");
return dataform.Target.create({ name, schema: schema + suffix });
} else {
return dataform.Target.create({
name: target,
schema: (defaultSchema || this.config.defaultSchema) + suffix
});
}
return dataform.Target.create({
name: target,
schema: (defaultSchema || this.config.defaultSchema) + suffix
});
}

public resolve(name: string): string {
const table = this.tables[name];
const operation = this.operations[name];

if (table && table.proto.type === "inline") {
// TODO: Pretty sure this is broken as the proto.query value may not
// be set yet as it happens during compilation. We should evalute the query here.
return `(${table.proto.query})`;
}
return this.adapter().resolveTarget(this.target(name));

const dataset = table || operation;
// TODO: We fall back to using the plain 'name' here for backwards compatibility with projects that use .sql files.
// In these projects, this session may not know about all actions (yet), and thus we need to fall back to assuming
// that the target *will* exist in the future. Once we break backwards compatibility with .sql files, we should remove
// the code that calls 'this.target(...)' below, and append a compile error if we can't find a dataset whose name is 'name'.
const target = dataset ? dataset.proto.target : this.target(name);
return this.adapter().resolveTarget(target);
}

public operate(name: string, queries?: OContextable<string | string[]>): Operation {
27 changes: 0 additions & 27 deletions core/table.ts
Original file line number Diff line number Diff line change
@@ -200,33 +200,6 @@ export class Table {
}
}

export class SqlxTableContext {
private table?: Table;

constructor(table: Table) {
this.table = table;
}

public self(): string {
return this.resolve(this.table.proto.name);
}

public ref(name: string) {
if (!name) {
const message = `Node name is not specified`;
this.table.session.compileError(new Error(message));
return "";
}

this.table.dependencies(name);
return this.resolve(name);
}

public resolve(name: string) {
return this.table.session.resolve(name);
}
}

export class TableContext {
private table?: Table;

Original file line number Diff line number Diff line change
@@ -5,4 +5,4 @@ config {

CREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)
---
DROP VIEW IF EXISTS someschema.someview
DROP VIEW IF EXISTS ${ref("override_schema_example")}
8 changes: 4 additions & 4 deletions tests/api/api.spec.ts
Original file line number Diff line number Diff line change
@@ -428,7 +428,7 @@ describe("@dataform/api", () => {
});

describe("init", () => {
it("init", async function () {
it("init", async function() {
this.timeout(30000);

// create temp directory
@@ -693,9 +693,9 @@ describe("@dataform/api", () => {
expect(exampleOperations.target).is.null;
expect(exampleOperations.queries).to.eql([
"\n\nCREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)\n",
"\nDROP VIEW IF EXISTS someschema.someview"
"\nDROP VIEW IF EXISTS `tada-analytics.override_schema.override_schema_example`\n"
]);
expect(exampleOperations.dependencies).to.eql(["example_inline"]);
expect(exampleOperations.dependencies).to.eql(["example_inline", "override_schema_example"]);

// Check example operation with output.
const exampleOperationWithOutput = graph.operations.find(
@@ -814,7 +814,7 @@ describe("@dataform/api", () => {
new Builder(graph, {}, { tables: [] }).build();
});

it("operation_refing", async function () {
it("operation_refing", async function() {
const expectedQueries = {
sample_1: 'create table "test_schema"."sample_1" as select 1 as sample_1',
sample_2: 'select * from "test_schema"."sample_1"'
16 changes: 8 additions & 8 deletions tests/core/core.spec.ts
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ const TEST_CONFIG: dataform.IProjectConfig = {

describe("@dataform/core", () => {
describe("publish", () => {
it("config", function () {
it("config", function() {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
const t = session
.publish("example", {
@@ -38,7 +38,7 @@ describe("@dataform/core", () => {
expect(t.postOps).deep.equals(["post_op"]);
});

it("config_context", function () {
it("config_context", function() {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
const t = session
.publish(
@@ -63,7 +63,7 @@ describe("@dataform/core", () => {
expect(t.postOps).deep.equals(["post_op"]);
});

it("validation_type_incremental", function () {
it("validation_type_incremental", function() {
const sessionSuccess = new Session(path.dirname(__filename), TEST_CONFIG);
sessionSuccess
.publish("exampleSuccess1", {
@@ -128,7 +128,7 @@ describe("@dataform/core", () => {
});
});

it("validation_type", function () {
it("validation_type", function() {
const sessionSuccess = new Session(path.dirname(__filename), TEST_CONFIG);
sessionSuccess.publish("exampleSuccess1", { type: "table" });
sessionSuccess.publish("exampleSuccess2", { type: "view" });
@@ -155,7 +155,7 @@ describe("@dataform/core", () => {
.that.matches(/Wrong type of table/);
});

it("validation_redshift_success", function () {
it("validation_redshift_success", function() {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("example_without_dist", {
redshift: {
@@ -183,7 +183,7 @@ describe("@dataform/core", () => {
.to.be.an("array").that.is.empty;
});

it("validation_redshift_fail", function () {
it("validation_redshift_fail", function() {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("example_absent_distKey", {
redshift: {
@@ -268,7 +268,7 @@ describe("@dataform/core", () => {
});
});

it("validation_type_inline", function () {
it("validation_type_inline", function() {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("a", { type: "table" }).query(_ => "select 1 as test");
session
@@ -490,7 +490,7 @@ describe("@dataform/core", () => {
});

describe("compilers", () => {
it("extract_blocks", function () {
it("extract_blocks", function() {
const TEST_SQL_FILE = `
/*js
var a = 1;
2 changes: 1 addition & 1 deletion version.bzl
Original file line number Diff line number Diff line change
@@ -1 +1 @@
DF_VERSION = "1.0.0-alpha.3"
DF_VERSION = "1.0.0-alpha.4"

0 comments on commit 1f4daea

Please sign in to comment.