diff --git a/.github/workflows/CI-CD-Squid-Explorer.yml b/.github/workflows/CI-CD-Squid-Explorer.yml index e26ba8fd2b..b31f865f64 100644 --- a/.github/workflows/CI-CD-Squid-Explorer.yml +++ b/.github/workflows/CI-CD-Squid-Explorer.yml @@ -54,7 +54,7 @@ jobs: with: file: idea/explorer/Dockerfile push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-explorer:${{ env.tag }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-explorer-latest:${{ env.tag }} build-squid-image: runs-on: ubuntu-latest @@ -87,7 +87,7 @@ jobs: with: file: idea/squid/Dockerfile push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-squid:${{ env.tag }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-squid-latest:${{ env.tag }} deploy-to-k8s: needs: @@ -108,10 +108,10 @@ jobs: run: | if [ "${{ github.ref }}" == "refs/heads/idea-release" ]; then echo "namespace=prod-idea" >> $GITHUB_ENV - echo "deployments=explorer squid-testnet squid-mainnet" >> $GITHUB_ENV + echo "deployments=explorer-latest squid-testnet-latest squid-mainnet-latest" >> $GITHUB_ENV else echo "namespace=dev-1" >> $GITHUB_ENV - echo "deployments=explorer squid-testnet" >> $GITHUB_ENV + echo "deployments=explorer-latest squid-testnet-latest" >> $GITHUB_ENV fi - name: Deploy to k8s diff --git a/idea/explorer/src/services/event.ts b/idea/explorer/src/services/event.ts index eb1e397a11..2430a5310e 100644 --- a/idea/explorer/src/services/event.ts +++ b/idea/explorer/src/services/event.ts @@ -24,13 +24,17 @@ export class EventService { } @Pagination() - async getEvents({ source, service, name, limit, offset }: ParamGetEvents): Promise> { + async getEvents({ source, parentId, service, name, limit, offset }: ParamGetEvents): Promise> { const builder = this._repo.createQueryBuilder('event'); if (source) { builder.andWhere('event.source = :source', { source }); } + if (parentId) { + builder.andWhere('event.parent_id = :parentId', { parentId }); + } + if (service) { builder.andWhere('event.service ILIKE :service', { service: `%${service.toLowerCase()}%` }); } diff --git a/idea/explorer/src/services/message.ts b/idea/explorer/src/services/message.ts index 326aea7a58..e61167d2f5 100644 --- a/idea/explorer/src/services/message.ts +++ b/idea/explorer/src/services/message.ts @@ -89,6 +89,7 @@ export class MessageService { async getMsgsFrom({ source, destination, + parentId, isInMailbox, limit, offset, @@ -105,6 +106,10 @@ export class MessageService { qb.andWhere('msg.destination = :destination', { destination }); } + if (parentId) { + qb.andWhere('msg.parent_id = :parentId', { parentId }); + } + if (isInMailbox) { qb.andWhere('msg.readReson = NULL').andWhere('msg.expiration != NULL'); } diff --git a/idea/explorer/src/types/requests/event.ts b/idea/explorer/src/types/requests/event.ts index acb2a658b4..bdcd4326cc 100644 --- a/idea/explorer/src/types/requests/event.ts +++ b/idea/explorer/src/types/requests/event.ts @@ -8,4 +8,5 @@ export interface ParamGetEvents extends ParamPagination, ParamGenesis { service?: string; name?: string; source?: string; + parentId?: string; } diff --git a/idea/explorer/src/types/requests/message.ts b/idea/explorer/src/types/requests/message.ts index 083ce4dd30..584851df85 100644 --- a/idea/explorer/src/types/requests/message.ts +++ b/idea/explorer/src/types/requests/message.ts @@ -23,6 +23,7 @@ export class ParamGetMsgsToProgram extends ParamPagination { export class ParamGetMsgsFromProgram extends ParamPagination { readonly destination?: string; readonly source?: string; + readonly parentId?: string; readonly isInMailbox?: boolean; readonly service?: string; readonly fn?: string; diff --git a/idea/indexer-db/src/entities/event.entity.ts b/idea/indexer-db/src/entities/event.entity.ts index e6f4f335ed..f34570b6eb 100644 --- a/idea/indexer-db/src/entities/event.entity.ts +++ b/idea/indexer-db/src/entities/event.entity.ts @@ -20,6 +20,9 @@ export class Event extends BaseEntity { @Column({ nullable: true }) public payload: string; + @Column({ nullable: true, name: 'parent_id' }) + public parentId: string; + @Column({ nullable: true }) public service?: string; diff --git a/idea/indexer-db/src/entities/message-from-program.entity.ts b/idea/indexer-db/src/entities/message-from-program.entity.ts index 7e895aaec7..ab11ac2b63 100644 --- a/idea/indexer-db/src/entities/message-from-program.entity.ts +++ b/idea/indexer-db/src/entities/message-from-program.entity.ts @@ -24,6 +24,9 @@ export class MessageFromProgram extends BaseEntity { @Column({ nullable: true }) public payload: string; + @Column({ nullable: true, name: 'parent_id' }) + public parentId: string; + @Column({ default: '0' }) public value: string; diff --git a/idea/squid/db/migrations/1725631344412-Data.js b/idea/squid/db/migrations/1725631344412-Data.js new file mode 100644 index 0000000000..dc51aa8f63 --- /dev/null +++ b/idea/squid/db/migrations/1725631344412-Data.js @@ -0,0 +1,13 @@ +module.exports = class Data1725704609634 { + name = 'Data1725704609634' + + async up(db) { + await db.query(`ALTER TABLE "message_from_program" ADD "parent_id" character varying`) + await db.query(`ALTER TABLE "event" ADD "parent_id" character varying`) + } + + async down(db) { + await db.query(`ALTER TABLE "message_from_program" DROP COLUMN "parent_id"`) + await db.query(`ALTER TABLE "event" DROP COLUMN "parent_id"`) + } +} diff --git a/idea/squid/package.json b/idea/squid/package.json index 8032ecdfd8..f29f264972 100644 --- a/idea/squid/package.json +++ b/idea/squid/package.json @@ -23,6 +23,7 @@ "dotenv": "16.4.5", "indexer-db": "workspace:^", "pg": "8.11.5", + "redis": "^4.7.0", "sails-js": "^0.1.9", "typeorm": "0.3.20" }, @@ -31,6 +32,7 @@ "@subsquid/substrate-typegen": "8.1.0", "@subsquid/typeorm-codegen": "2.0.0", "@types/node": "20.12.7", + "@types/redis": "^4.0.11", "ts-node-dev": "^2.0.0", "typescript": "5.4.5" } diff --git a/idea/squid/src/config.ts b/idea/squid/src/config.ts index 3d0aba7a5f..4003c67497 100644 --- a/idea/squid/src/config.ts +++ b/idea/squid/src/config.ts @@ -10,4 +10,10 @@ export const config = { fromBlock: parseInt(process.env.FROM_BLOCK || '0'), toBlock: parseInt(process.env.TO_BLOCK) || undefined, }, + redis: { + host: process.env.REDIS_HOST || '127.0.0.1', + port: parseInt(process.env.REDIS_PORT) || 6379, + user: process.env.REDIS_USER || '', + password: process.env.REDIS_PASSWORD || '', + }, }; diff --git a/idea/squid/src/event.route.ts b/idea/squid/src/event.route.ts index 3e15065afb..ce6cd876d0 100644 --- a/idea/squid/src/event.route.ts +++ b/idea/squid/src/event.route.ts @@ -31,7 +31,10 @@ export interface IHandleEventProps { block: Block; } -const callHandlers: Array<{ pattern: (obj: any) => boolean; handler: (args: IHandleCallProps) => void }> = [ +const callHandlers: Array<{ + pattern: (obj: any) => boolean; + handler: (args: IHandleCallProps) => void; +}> = [ { pattern: isUploadProgram, handler: handleUploadProgram }, { pattern: isSendMessageCall, handler: handleSendMessageCall }, { pattern: isVoucherCall, handler: handleVoucherCall }, @@ -72,8 +75,11 @@ export async function handleUserMessageSent({ event, common, tempState }: IHandl value: event.args.message.value, replyToMessageId: event.args.message.details?.to || null, expiration: event.args.expirtaion || null, - exitCode: event.args.message.details?.code?.__kind === 'Success' ? 0 : 1, + exitCode: !event.args.message.details?.code ? null : event.args.message.details.code.__kind === 'Success' ? 0 : 1, }); + + msg.parentId = msg.replyToMessageId ? msg.replyToMessageId : await tempState.getMessageId(msg.id); + if (event.args.message.destination === ZERO_ADDRESS) { tempState.addEvent(msg); } else { @@ -140,7 +146,12 @@ export async function handleCodeChanged({ event, common, tempState }: IHandleEve } export async function handleMessagesDispatched({ event, tempState }: IHandleEventProps) { - await Promise.all(event.args.statuses.map((s) => tempState.setDispatchedStatus(s[0], s[1].__kind))); + await Promise.all( + event.args.statuses.map((s) => { + tempState.removeParentMsgId(s[0]); + return tempState.setDispatchedStatus(s[0], s[1].__kind); + }), + ); } const reasons = { diff --git a/idea/squid/src/main.ts b/idea/squid/src/main.ts index 58c7bc72d6..0fd4b0d03b 100644 --- a/idea/squid/src/main.ts +++ b/idea/squid/src/main.ts @@ -19,10 +19,13 @@ import { handleUserMessageSent, IHandleEventProps, } from './event.route'; +import { createClient, RedisClientType } from 'redis'; +import { config } from './config'; +import { GearApi } from '@gear-js/api'; let tempState: TempState; -const callHandlers: Array<{ pattern: (obj: any) => boolean; handler: (args: IHandleEventProps) => Promise }> = [ +const eventHandlers: Array<{ pattern: (obj: any) => boolean; handler: (args: IHandleEventProps) => Promise }> = [ { pattern: isMessageQueued, handler: handleMessageQueued }, { pattern: isUserMessageSent, handler: handleUserMessageSent }, { pattern: isProgramChanged, handler: handleProgramChanged }, @@ -42,7 +45,7 @@ const handler = async (ctx: ProcessorContext) => { }; for (const event of block.events) { - const { handler } = callHandlers.find(({ pattern }) => pattern(event)); + const { handler } = eventHandlers.find(({ pattern }) => pattern(event)); if (!handler) { continue; @@ -55,12 +58,27 @@ const handler = async (ctx: ProcessorContext) => { await tempState.save(); }; -const main = async () => { - tempState = new TempState(); +interface RedisClient extends RedisClientType {} + +const main = async (api: GearApi) => { + const redisClient: RedisClient = createClient({ + username: config.redis.user, + password: config.redis.password, + socket: { + host: config.redis.host, + port: config.redis.port, + }, + }); + await redisClient.connect(); + + tempState = new TempState(redisClient, api.genesisHash.toHex()); + api.disconnect(); processor.run(new TypeormDatabase({ supportHotBlocks: true }), handler); }; -main().catch((e) => { - console.error(e); - process.exit(1); -}); +GearApi.create({ providerAddress: config.squid.rpc }) + .then(main) + .catch((e) => { + console.error(e); + process.exit(1); + }); diff --git a/idea/squid/src/temp-state.ts b/idea/squid/src/temp-state.ts index ff013f3297..a0539e9a3a 100644 --- a/idea/squid/src/temp-state.ts +++ b/idea/squid/src/temp-state.ts @@ -17,6 +17,8 @@ import { } from './model'; import { Block, ProcessorContext } from './processor'; import { MessageStatus } from './common'; +import { RedisClientType } from '@redis/client'; +import { findChildMessageId } from './util'; const gearProgramModule = xxhashAsHex('GearProgram', 128); const programStorageMethod = xxhashAsHex('ProgramStorage', 128); @@ -49,22 +51,28 @@ export class TempState { private messagesFromProgram: Map; private messagesToProgram: Map; private events: Map; + private cachedMessages: { [key: string]: number }; + private genesisHash: string; private _ctx: ProcessorContext; private _metadata: Metadata; private _registry: TypeRegistry; private _specVersion: number; private _programStorageTy: string; + private _redis: RedisClientType; - constructor() { + constructor(redisClient: RedisClientType, genesisHash: string) { + this._redis = redisClient; + this.genesisHash = genesisHash; this.programs = new Map(); this.codes = new Map(); this.messagesFromProgram = new Map(); this.messagesToProgram = new Map(); this.events = new Map(); this.newPrograms = new Set(); + this.cachedMessages = {}; } - newState(ctx: ProcessorContext) { + async newState(ctx: ProcessorContext) { this._ctx = ctx; this.programs.clear(); this.codes.clear(); @@ -72,6 +80,14 @@ export class TempState { this.messagesToProgram.clear(); this.events.clear(); this.newPrograms.clear(); + + this._redis.on('error', (error) => this._ctx.log.error(error)); + + const temp = Object.entries(await this._redis.hGetAll(this.genesisHash)); + this.cachedMessages = {}; + temp.forEach(([key, value]) => { + this.cachedMessages[key] = Number(value); + }); } addProgram(program: Program) { @@ -89,6 +105,8 @@ export class TempState { msg.service = service; msg.fn = name; + this.saveParentMsgId(msg.id); + this.messagesToProgram.set(msg.id, msg); } @@ -114,6 +132,7 @@ export class TempState { blockHash: msg.blockHash, blockNumber: msg.blockNumber, id: msg.id, + parentId: msg.parentId, source: msg.source, payload: msg.payload, service, @@ -296,11 +315,37 @@ export class TempState { 'Data saved', ); } + + await this._redis.del(this.genesisHash); + if (Object.keys(this.cachedMessages).length > 0) { + await this._redis.hSet(this.genesisHash, this.cachedMessages); + } } catch (error) { this._ctx.log.error({ error: error.message, stack: error.stack }, 'Failed to save data'); throw error; } } + + saveParentMsgId(parentId: string, nonce: number = 0) { + this.cachedMessages[parentId] = nonce; + return parentId; + } + + removeParentMsgId(parentId: string) { + delete this.cachedMessages[parentId]; + } + + async getMessageId(childId: string) { + const finder = Object.entries(this.cachedMessages).map(([parentId, nonce]) => { + return findChildMessageId(parentId, childId, Number(nonce)); + }); + + return Promise.any(finder) + .then(({ parentId, nonce }) => { + return this.saveParentMsgId(parentId, nonce); + }) + .catch(() => null); + } } const getAllNeccesaryTypes = (metadata: Metadata, tyindex: SiLookupTypeId | number): Record => { diff --git a/idea/squid/src/util.ts b/idea/squid/src/util.ts index 9a5855e163..f80fbeb5ad 100644 --- a/idea/squid/src/util.ts +++ b/idea/squid/src/util.ts @@ -1,14 +1,15 @@ import { CUploadCode, CUploadProgram, CVoucherCall, isUploadCode, isUploadProgram } from './types/calls'; -import { getGrReply } from '@gear-js/api'; -import { u8aToHex } from '@polkadot/util'; +import { getGrReply, CreateType } from '@gear-js/api'; +import { u8aToHex, u8aToU8a, u8aConcat, stringToU8a } from '@polkadot/util'; +import { blake2AsHex } from '@polkadot/util-crypto'; export async function getMetahash(call: CUploadCode | CUploadProgram | CVoucherCall): Promise { const code = isUploadCode(call) || isUploadProgram(call) ? call.args.code : call.args.call.__kind === 'UploadCode' - ? call.args.call.code - : null; + ? call.args.call.code + : null; if (code) { let metahash: Uint8Array; @@ -24,3 +25,20 @@ export async function getMetahash(call: CUploadCode | CUploadProgram | CVoucherC return null; } + +const prefix = stringToU8a('outgoing'); +const nonces = Array.from({ length: 512 }, (_v, i) => CreateType.create('u32', i).toU8a()); + +export async function findChildMessageId(parentId: string, idToFind: string, startNonce: number = 0) { + const msgId = u8aToU8a(parentId); + + for (let i = startNonce; i < nonces.length; i++) { + const childId = blake2AsHex(u8aConcat(prefix, msgId, nonces[i])); + + if (childId === idToFind) { + return { parentId, nonce: i }; + } + } + + throw Error('Child id not found'); +} diff --git a/idea/squid/tsconfig.json b/idea/squid/tsconfig.json index 03553c22ba..bc31f23382 100644 --- a/idea/squid/tsconfig.json +++ b/idea/squid/tsconfig.json @@ -5,7 +5,10 @@ "emitDecoratorMetadata": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, - "strict": false + "strict": false, + "paths": { + "@/*": ["./src/*"] + } }, "include": ["src"] } diff --git a/yarn.lock b/yarn.lock index e596c5315f..36c5536e71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6597,6 +6597,15 @@ __metadata: languageName: node linkType: hard +"@types/redis@npm:^4.0.11": + version: 4.0.11 + resolution: "@types/redis@npm:4.0.11" + dependencies: + redis: "*" + checksum: 4b2d252368a7dc78738a98a8bcc817f92ca097511a901427628566f24f53998a2b1b524a85dea6a19307d4a95b55bd7d5a6ad7cc944ae8f39e54ca2f28ddf893 + languageName: node + linkType: hard + "@types/resolve@npm:1.17.1": version: 1.17.1 resolution: "@types/resolve@npm:1.17.1" @@ -12188,9 +12197,11 @@ __metadata: "@subsquid/typeorm-migration": 1.3.0 "@subsquid/typeorm-store": 1.5.1 "@types/node": 20.12.7 + "@types/redis": ^4.0.11 dotenv: 16.4.5 indexer-db: "workspace:^" pg: 8.11.5 + redis: ^4.7.0 sails-js: ^0.1.9 ts-node-dev: ^2.0.0 typeorm: 0.3.20 @@ -16615,7 +16626,7 @@ __metadata: languageName: node linkType: hard -"redis@npm:^4.6.15, redis@npm:^4.6.8": +"redis@npm:*, redis@npm:^4.6.15, redis@npm:^4.6.8, redis@npm:^4.7.0": version: 4.7.0 resolution: "redis@npm:4.7.0" dependencies: