From ba1c6ea158de564456365b8613245ffcba702a20 Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 5 Dec 2023 13:38:00 -0500 Subject: [PATCH 1/2] Remove old tables and update the analytics cdk stack --- common/data/src/providers/schema.ts | 10 ++-- common/data/src/schemas/event.schema.json | 55 ++++++------------- infrastructure/cdk/src/providers/analytics.ts | 37 +++++++------ infrastructure/cdk/test/analytics.test.ts | 16 +----- 4 files changed, 43 insertions(+), 75 deletions(-) diff --git a/common/data/src/providers/schema.ts b/common/data/src/providers/schema.ts index 83db4a2b8..59a409e9d 100644 --- a/common/data/src/providers/schema.ts +++ b/common/data/src/providers/schema.ts @@ -41,10 +41,8 @@ export class Schema { let type: GlueType = glue.Schema[typeKey] - if (name.endsWith("_at")) type = glue.Schema.TIMESTAMP - if (name.endsWith("_balance")) type = glue.Schema.BIG_INT - if (name == "amount") type = glue.Schema.BIG_INT - if (name === "price") type = glue.Schema.FLOAT + if (name == "timestamp") type = glue.Schema.TIMESTAMP + if (name === "amount") type = glue.Schema.FLOAT const comment = property.description return { name, type, comment } @@ -84,10 +82,10 @@ export class Schema { const comment = property.description if (comment.includes("PK")) column += " PRIMARY KEY" - + const defaultValue = property.default if (defaultValue !== undefined) column += ` DEFAULT ${defaultValue}` - + return column }) diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index 9a8705bae..699059a6f 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -4,66 +4,43 @@ "$comment": "analytics", "title": "Event", "type": "object", + "description": "Event schema covers all events emitted by the Casimir contracts (e.g. Factory, Manager, ..)", "properties": { - "chain": { - "type": "string", - "description": "The chain which the event belongs to (e.g. iotex, ethereum)" - }, "network": { "type": "string", - "description": "Network type (e.g. mainnet, testnet)" + "description": "Network type (e.g. mainnet, goerli)" }, - "provider": { - "type": "string", - "description": "The provider used to source the event (e.g. infura, consensus)" - }, - "type": { - "type": "string", - "description": "The type of event (e.g. block, transaction)" - }, - "height": { + "block_number": { "type": "integer", - "description": "The height of the block the event belongs to" + "description": "Block number" }, - "block": { + "block_hash": { "type": "string", - "description": "The block hash" + "description": "Block hash" }, - "transaction": { + "tx_hash": { "type": "string", "description": "The transaction hash" }, - "received_at": { + "timestamp": { "type": "integer", - "description": "Timestamp of the event in unix format" + "description": "The timestamp of the log" }, - "sender": { + "contract": { "type": "string", - "description": "The sender's address" + "description": "The contract address which emitted the event" }, - "recipient": { + "event": { "type": "string", - "description": "The recipient's address" + "description": "Type of casimir contract event (e.g. StakeDeposit, StakeRebalanced, WithrawalFulfilled)" }, - "sender_balance": { - "type": "string", - "description": "The sender's balance at the time of the event" - }, - "recipient_balance": { + "sender": { "type": "string", - "description": "The recipient's balance at the time of the event" + "description": "The sender's address" }, "amount": { - "type": "string", + "type": "number", "description": "The amount transferred in the event" - }, - "price": { - "type": "string", - "description": "The exchange price of the coin at the time of the event" - }, - "gas_fee": { - "type": "integer", - "description": "The gas fee paid for the transaction" } } } \ No newline at end of file diff --git a/infrastructure/cdk/src/providers/analytics.ts b/infrastructure/cdk/src/providers/analytics.ts index 26e5c0e77..61904b3c5 100644 --- a/infrastructure/cdk/src/providers/analytics.ts +++ b/infrastructure/cdk/src/providers/analytics.ts @@ -2,16 +2,18 @@ import { Construct } from "constructs" import * as cdk from "aws-cdk-lib" import * as s3 from "aws-cdk-lib/aws-s3" import * as glue from "@aws-cdk/aws-glue-alpha" -import { Schema, eventSchema, actionSchema } from "@casimir/data" +import { Schema, eventSchema } from "@casimir/data" import { kebabCase, pascalCase, snakeCase } from "@casimir/format" import { Config } from "./config" import { AnalyticsStackProps } from "../interfaces/StackProps" +import { CfnWorkGroup } from "aws-cdk-lib/aws-athena" /** * Data analytics stack */ export class AnalyticsStack extends cdk.Stack { public readonly name = pascalCase("analytics") + public readonly workGroupName: string constructor(scope: Construct, id: string, props: AnalyticsStackProps) { super(scope, id, props) @@ -19,24 +21,33 @@ export class AnalyticsStack extends cdk.Stack { const config = new Config() const eventColumns = new Schema(eventSchema).getGlueColumns() - const actionColumns = new Schema(actionSchema).getGlueColumns() - const database = new glue.Database(this, config.getFullStackResourceName(this.name, "database"), { - databaseName: snakeCase(config.getFullStackResourceName(this.name, "database")) + const database = new glue.Database(this, config.getFullStackResourceName(this.name, "database", config.dataVersion), { + databaseName: snakeCase(config.getFullStackResourceName(this.name, "database", config.dataVersion)), }) const eventBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, "event-bucket", config.dataVersion), { bucketName: kebabCase(config.getFullStackResourceName(this.name, "event-bucket", config.dataVersion)) }) - const actionBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, "action-bucket", config.dataVersion), { - bucketName: kebabCase(config.getFullStackResourceName(this.name, "action-bucket", config.dataVersion)) - }) - - new s3.Bucket(this, config.getFullStackResourceName(this.name, "output-bucket", config.dataVersion), { + const outputBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, "output-bucket", config.dataVersion), { bucketName: kebabCase(config.getFullStackResourceName(this.name, "output-bucket", config.dataVersion)) }) + const workGroup = new CfnWorkGroup(this, config.getFullStackResourceName(this.name, "workgroup", config.dataVersion), { + name: config.getFullStackResourceName(this.name, "workgroup", config.dataVersion), + recursiveDeleteOption: true, + state: "ENABLED", + workGroupConfiguration: { + resultConfiguration: { + outputLocation: `s3://${outputBucket.bucketName}/`, + }, + }, + tags: [{ key: "version", value: config.dataVersion.toString() }], + }) + + this.workGroupName = workGroup.name + new glue.Table(this, config.getFullStackResourceName(this.name, "event-table", config.dataVersion), { database: database, tableName: snakeCase(config.getFullStackResourceName(this.name, "event-table", config.dataVersion)), @@ -44,13 +55,5 @@ export class AnalyticsStack extends cdk.Stack { columns: eventColumns, dataFormat: glue.DataFormat.JSON, }) - - new glue.Table(this, config.getFullStackResourceName(this.name, "action-table", config.dataVersion), { - database: database, - tableName: snakeCase(config.getFullStackResourceName(this.name, "action-table", config.dataVersion)), - bucket: actionBucket, - columns: actionColumns, - dataFormat: glue.DataFormat.JSON, - }) } } \ No newline at end of file diff --git a/infrastructure/cdk/test/analytics.test.ts b/infrastructure/cdk/test/analytics.test.ts index 5f397f689..9c495ed8d 100644 --- a/infrastructure/cdk/test/analytics.test.ts +++ b/infrastructure/cdk/test/analytics.test.ts @@ -2,7 +2,7 @@ import * as cdk from "aws-cdk-lib" import * as assertions from "aws-cdk-lib/assertions" import { Config } from "../src/providers/config" import { AnalyticsStack } from "../src/providers/analytics" -import { Schema, eventSchema, actionSchema } from "@casimir/data" +import { Schema, eventSchema, } from "@casimir/data" test("Analytics stack created", () => { const config = new Config() @@ -30,17 +30,7 @@ test("Analytics stack created", () => { expect(columnName).toEqual(name) } - const actionTable = Object.keys(resource).filter(key => key.includes("ActionTable"))[0] - const actionColumns = resource[actionTable].Properties.TableInput.StorageDescriptor.Columns - const actionGlueSchema = new Schema(actionSchema).getGlueColumns() + const workgroup = analyticsTemplate.findResources("AWS::Athena::WorkGroup") - - for (const column of actionColumns) { - const { Name: name, Type: type } = column - const columnName = Object.keys(actionSchema.properties).filter(key => key === name)[0] - const columnType = actionGlueSchema.filter(key => key.name === name)[0].type.inputString - - expect(columnType).toEqual(type) - expect(columnName).toEqual(name) - } + expect(workgroup).toBeDefined() }) \ No newline at end of file From e0d42b5d40211f7f1102b7ba602153579b8e330f Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 5 Dec 2023 17:25:44 -0500 Subject: [PATCH 2/2] Change amount type to string and update cdk test --- common/data/src/providers/schema.ts | 1 - common/data/src/schemas/event.schema.json | 2 +- infrastructure/cdk/src/providers/analytics.ts | 7 ++----- infrastructure/cdk/test/analytics.test.ts | 5 +++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/common/data/src/providers/schema.ts b/common/data/src/providers/schema.ts index 59a409e9d..3ecd9313a 100644 --- a/common/data/src/providers/schema.ts +++ b/common/data/src/providers/schema.ts @@ -42,7 +42,6 @@ export class Schema { let type: GlueType = glue.Schema[typeKey] if (name == "timestamp") type = glue.Schema.TIMESTAMP - if (name === "amount") type = glue.Schema.FLOAT const comment = property.description return { name, type, comment } diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index 699059a6f..fc6821523 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -39,7 +39,7 @@ "description": "The sender's address" }, "amount": { - "type": "number", + "type": "string", "description": "The amount transferred in the event" } } diff --git a/infrastructure/cdk/src/providers/analytics.ts b/infrastructure/cdk/src/providers/analytics.ts index 61904b3c5..c2f8f4bb9 100644 --- a/infrastructure/cdk/src/providers/analytics.ts +++ b/infrastructure/cdk/src/providers/analytics.ts @@ -13,7 +13,6 @@ import { CfnWorkGroup } from "aws-cdk-lib/aws-athena" */ export class AnalyticsStack extends cdk.Stack { public readonly name = pascalCase("analytics") - public readonly workGroupName: string constructor(scope: Construct, id: string, props: AnalyticsStackProps) { super(scope, id, props) @@ -34,8 +33,8 @@ export class AnalyticsStack extends cdk.Stack { bucketName: kebabCase(config.getFullStackResourceName(this.name, "output-bucket", config.dataVersion)) }) - const workGroup = new CfnWorkGroup(this, config.getFullStackResourceName(this.name, "workgroup", config.dataVersion), { - name: config.getFullStackResourceName(this.name, "workgroup", config.dataVersion), + new CfnWorkGroup(this, config.getFullStackResourceName(this.name, "workGroup", config.dataVersion), { + name: config.getFullStackResourceName(this.name, "workGroup", config.dataVersion), recursiveDeleteOption: true, state: "ENABLED", workGroupConfiguration: { @@ -46,8 +45,6 @@ export class AnalyticsStack extends cdk.Stack { tags: [{ key: "version", value: config.dataVersion.toString() }], }) - this.workGroupName = workGroup.name - new glue.Table(this, config.getFullStackResourceName(this.name, "event-table", config.dataVersion), { database: database, tableName: snakeCase(config.getFullStackResourceName(this.name, "event-table", config.dataVersion)), diff --git a/infrastructure/cdk/test/analytics.test.ts b/infrastructure/cdk/test/analytics.test.ts index 9c495ed8d..b4c56d074 100644 --- a/infrastructure/cdk/test/analytics.test.ts +++ b/infrastructure/cdk/test/analytics.test.ts @@ -31,6 +31,7 @@ test("Analytics stack created", () => { } const workgroup = analyticsTemplate.findResources("AWS::Athena::WorkGroup") - - expect(workgroup).toBeDefined() + const outputBucket = Object.keys(analyticsTemplate.findResources("AWS::S3::Bucket")).filter(key => key.includes("OutputBucket"))[0] + const workgroupRef = workgroup[Object.keys(workgroup)[0]].Properties.WorkGroupConfiguration.ResultConfiguration.OutputLocation["Fn::Join"][1][1]["Ref"] + expect(workgroupRef).toEqual(outputBucket) }) \ No newline at end of file