Skip to content

Commit

Permalink
update modules
Browse files Browse the repository at this point in the history
  • Loading branch information
@dforsber
dforsber committed Jul 28, 2023
1 parent 286e388 commit ad3b080
Show file tree
Hide file tree
Showing 9 changed files with 2,775 additions and 2,275 deletions.
43 changes: 21 additions & 22 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,32 +41,31 @@
}
},
"dependencies": {
"@aws-sdk/client-glue": "^3.2.0",
"@aws-sdk/client-s3": "^3.2.0",
"@aws-sdk/client-glue": "^3.379.1",
"@aws-sdk/client-s3": "^3.379.1",
"bunyan": "^1.8.15",
"merge-stream": "^2.0.0",
"node-sql-parser": "^4.3.0",
"sqlite3": "^5.0.0"
"node-sql-parser": "^4.8.0",
"sqlite3": "^5.1.6"
},
"devDependencies": {
"@aws-sdk/types": "3.78.0",
"@types/bunyan": "^1.8.6",
"@types/jest": "^27.5.1",
"@types/bunyan": "^1.8.8",
"@types/jest": "^29.5.3",
"@types/merge-stream": "^1.1.2",
"@types/node": "^17.0.34",
"@types/sqlite3": "^3.1.6",
"@typescript-eslint/eslint-plugin": "^5.25.0",
"@typescript-eslint/parser": "^5.25.0",
"eslint": "^8.15.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-jest": "^26.2.2",
"eslint-plugin-prettier": "^4.0.0",
"husky": "^8.0.1",
"jest": "^28.1.0",
"onchange": "^7.0.2",
"prettier": "^2.0.5",
"rimraf": "^3.0.2",
"ts-jest": "^28.0.2",
"typescript": "^4.1.3"
"@types/node": "^20.4.5",
"@types/sqlite3": "^3.1.8",
"@typescript-eslint/eslint-plugin": "^6.2.0",
"@typescript-eslint/parser": "^6.2.0",
"eslint": "^8.45.0",
"eslint-config-prettier": "^8.9.0",
"eslint-plugin-jest": "^27.2.3",
"eslint-plugin-prettier": "^5.0.0",
"husky": "^8.0.3",
"jest": "^29.6.2",
"onchange": "^7.1.0",
"prettier": "^3.0.0",
"rimraf": "^5.0.1",
"ts-jest": "^29.1.1",
"typescript": "^5.1.6"
}
}
81 changes: 81 additions & 0 deletions src/mappers/__snapshots__/glueTableToS3Keys.mapper.test.ts.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`When fetching partitioning information correctly identifies table information (CSV) 1`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"CSV": {},
"CompressionType": "GZIP",
},
"PartitionColumns": [],
}
`;

exports[`When fetching partitioning information correctly identifies table information (CSV) 2`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"CSV": {},
"CompressionType": "GZIP",
},
"PartitionColumns": [],
}
`;

exports[`When fetching partitioning information correctly identifies table information (JSON) 1`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"JSON": {
"Type": "DOCUMENT",
},
},
"PartitionColumns": [],
}
`;

exports[`When fetching partitioning information correctly identifies table information (JSON) 2`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"JSON": {
"Type": "DOCUMENT",
},
},
"PartitionColumns": [],
}
`;

exports[`When fetching partitioning information correctly identifies table information (Parquet) 1`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"Parquet": {},
},
"PartitionColumns": [
"ssl_protocol",
"elb_response_code",
],
}
`;

exports[`When fetching partitioning information correctly identifies table information (Parquet) 2`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": {
"Parquet": {},
},
"PartitionColumns": [
"ssl_protocol",
"elb_response_code",
],
}
`;

exports[`When fetching partitioning information correctly identifies table information (unknown) 1`] = `
{
"Bucket": "dummy-test-bucket",
"InputSerialization": undefined,
"PartitionColumns": [],
}
`;
80 changes: 7 additions & 73 deletions src/mappers/glueTableToS3Keys.mapper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,31 +147,9 @@ describe("When fetching partitioning information", () => {
it("correctly identifies table information (Parquet)", async () => {
mapper = new GlueTableToS3Key({ glue, s3, databaseName, tableName: "partitioned_and_bucketed_elb_logs_parquet" });
let info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"Parquet": Object {},
},
"PartitionColumns": Array [
"ssl_protocol",
"elb_response_code",
],
}
`);
expect(info).toMatchSnapshot();
info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"Parquet": Object {},
},
"PartitionColumns": Array [
"ssl_protocol",
"elb_response_code",
],
}
`);
expect(info).toMatchSnapshot();
expect(glueGetTableCalled).toEqual(1);
expect(glueGetPartitionsCalled).toEqual(0);
expect(s3ListObjectsV2Called).toEqual(0);
Expand All @@ -180,27 +158,9 @@ describe("When fetching partitioning information", () => {
it("correctly identifies table information (CSV)", async () => {
mapper = new GlueTableToS3Key({ glue, s3, databaseName, tableName: "bucketed_elb_logs" });
let info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"CSV": Object {},
"CompressionType": "GZIP",
},
"PartitionColumns": Array [],
}
`);
expect(info).toMatchSnapshot();
info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"CSV": Object {},
"CompressionType": "GZIP",
},
"PartitionColumns": Array [],
}
`);
expect(info).toMatchSnapshot();
expect(glueGetTableCalled).toEqual(1);
expect(glueGetPartitionsCalled).toEqual(0);
expect(s3ListObjectsV2Called).toEqual(0);
Expand All @@ -209,41 +169,15 @@ describe("When fetching partitioning information", () => {
it("correctly identifies table information (unknown)", async () => {
mapper = new GlueTableToS3Key({ glue, s3, databaseName, tableName: "bucketed_elb_logs_unsupported_serde" });
const info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": undefined,
"PartitionColumns": Array [],
}
`);
expect(info).toMatchSnapshot();
});

it("correctly identifies table information (JSON)", async () => {
mapper = new GlueTableToS3Key({ glue, s3, databaseName, tableName: "bucketed_elb_logs_from_partitioned_2_json" });
let info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"JSON": Object {
"Type": "DOCUMENT",
},
},
"PartitionColumns": Array [],
}
`);
expect(info).toMatchSnapshot();
info = await mapper.getTableInfo();
expect(info).toMatchInlineSnapshot(`
Object {
"Bucket": "dummy-test-bucket",
"InputSerialization": Object {
"JSON": Object {
"Type": "DOCUMENT",
},
},
"PartitionColumns": Array [],
}
`);
expect(info).toMatchSnapshot();
expect(glueGetTableCalled).toEqual(1);
expect(glueGetPartitionsCalled).toEqual(0);
expect(s3ListObjectsV2Called).toEqual(0);
Expand Down
26 changes: 26 additions & 0 deletions src/s3-selectable/__snapshots__/s3-selectable-nonclass.test.ts.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`Non-class based s3selectable returns correct results valid output 1`] = `
[
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":1,"value":"test1"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
"{"id":2,"value":"test2"}",
]
`;
Loading

0 comments on commit ad3b080

Please sign in to comment.