feat(idea-backend): introduce new indexer based on subsquid (#1597)

Co-authored-by: sergey filyanin <[email protected]>

.github/workflows/CI-CD-Squid-Explorer.yml

@@ -1,8 +1,8 @@
-name: 'Deploy new squid and indexer to k8s stage'
+name: 'Deploy new squid and indexer to k8s'
 
 on:
   push:
-    branches: 
+    branches:
       - 'main'
       - 'idea-release'
     paths:
@@ -88,7 +88,7 @@ jobs:
           tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-squid:${{ env.tag }}
 
   deploy-to-k8s:
-    needs: 
+    needs:
       - build-explorer-image
       - build-squid-image
     runs-on: ubuntu-latest
@@ -106,10 +106,10 @@ jobs:
         run: |
           if [ "${{ github.ref }}" == "refs/heads/idea-release" ]; then
             echo "namespace=prod-idea" >> $GITHUB_ENV
-            echo "deployments=explorer-mainnet squid-mainnet" >> $GITHUB_ENV
+            echo "deployments=explorer squid-mainnet squid-testnet" >> $GITHUB_ENV
           else
             echo "namespace=dev-1" >> $GITHUB_ENV
-            echo "deployments=explorer-testnet squid-testnet" >> $GITHUB_ENV
+            echo "deployments=explorer squid-testnet" >> $GITHUB_ENV
 
       - name: Deploy to k8s
         uses: sergeyfilyanin/kubectl-aws-eks@master

.gitignore

@@ -10,3 +10,8 @@ node_modules/
 **/dist/
 .vscode/
 idea/tmp/
+
+.env
+idea/squid/lib/
+idea/indexer-db/lib/
+.spec.json

gear-js.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+command="$1"
+pkg="$2"
+
+SQUID_TYPEORM_MIGRATION_BIN="node_modules/.bin/squid-typeorm-migration"
+
+SQUID_PATH="idea/squid"
+EXPLORER_PATH="idea/explorer"
+
+
+if [ "$command" == "install" ]; then
+    echo "Installing dependencies"
+    yarn install
+elif [ "$command" == "build" ]; then
+    echo "Building $pkg"
+    yarn build:$pkg
+elif [ "$command" = "run" ]; then
+    echo "Running squid"
+    case $pkg in
+        "squid")
+            cd $SQUID_PATH
+            if [ -f "$SQUID_TYPEORM_MIGRATION_BIN" ]; then
+                node $SQUID_TYPEORM_MIGRATION_BIN apply
+            else
+                node ../../$SQUID_TYPEORM_MIGRATION_BIN apply
+            fi
+            node lib/main.js
+            ;;
+        "explorer")
+            cd $EXPLORER_PATH
+            node dist/main.js
+            ;;
+        *)
+            echo "Invalid package"
+            ;;
+    esac
+else
+    echo "Invalid command"
+fi

idea/explorer/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:20-alpine
+
+WORKDIR /src
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .yarn .yarn
+COPY .yarnrc.yml .
+COPY gear-js.sh .
+
+
+COPY idea/indexer-db ./idea/indexer-db
+COPY idea/explorer ./idea/explorer
+COPY idea/common ./idea/common
+
+RUN yarn install
+
+RUN ./gear-js.sh build common
+RUN ./gear-js.sh build indexer-db
+RUN ./gear-js.sh build explorer
+
+CMD ["/bin/sh", "-c", "/src/gear-js.sh run explorer"]

idea/explorer/README.md

@@ -0,0 +1 @@
+# indexer-gateway

idea/explorer/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "explorer",
+  "packageManager": "[email protected]",
+  "dependencies": {
+    "@gear-js/common": "workspace:^",
+    "class-validator": "0.14.1",
+    "cron": "^3.1.7",
+    "dotenv": "^16.4.5",
+    "express": "4.19.2",
+    "indexer-db": "workspace:^",
+    "nanoid": "^5.0.7",
+    "pg": "8.12.0",
+    "redis": "^4.6.15",
+    "reflect-metadata": "^0.2.2"
+  },
+  "devDependencies": {
+    "@types/express": "4.17.21",
+    "@types/node": "20.14.2",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "5.5.3"
+  },
+  "scripts": {
+    "build": "rm -rf dist && tsc",
+    "dev": "ts-node-dev src/main.ts",
+    "start": "node dist/main.js"
+  }
+}

idea/explorer/src/config.ts

@@ -0,0 +1,23 @@
+import * as dotenv from 'dotenv';
+
+dotenv.config();
+
+const getEnv = (key: string, default_?: string): string => {
+  const env = process.env[key] || default_;
+
+  if (env === undefined) {
+    throw new Error(`Missing environment variable ${key}`);
+  }
+
+  return env;
+};
+
+export const config = {
+  spec: process.env.SPEC_PATH,
+  redis: {
+    user: getEnv('REDIS_USER', ''),
+    password: getEnv('REDIS_PASSWORD', ''),
+    host: getEnv('REDIS_HOST'),
+    port: getEnv('REDIS_PORT'),
+  },
+};

idea/explorer/src/decorators/index.ts

@@ -0,0 +1,4 @@
+export * from './pagination';
+export * from './validation';
+export * from './method';
+export * from './required';

idea/explorer/src/decorators/method.ts

@@ -0,0 +1,80 @@
+import { GenesisNotFound, MethodNotFound, NetworkNotSupported } from '../errors';
+import { JsonRpcRequest, JsonRpcResponse } from '../types';
+
+type Constructor<T = any> = new (...args: any[]) => T;
+
+const rpcMethods: Record<string, (...args: any[]) => Promise<void>> = {};
+
+export function JsonRpcMethod(name: string) {
+  return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+    rpcMethods[name] = descriptor.value;
+  };
+}
+
+export interface IJsonRpc {
+  _getMethod(name: string): (...args: any[]) => Promise<void>;
+  handleRequest({ method, params, id }: JsonRpcRequest): Promise<JsonRpcResponse | JsonRpcResponse[]>;
+}
+
+export class JsonRpcBase implements IJsonRpc {
+  _getMethod(name: string): (...args: any[]) => Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  handleRequest({ method, params, id }: JsonRpcRequest): Promise<JsonRpcResponse | JsonRpcResponse[]> {
+    throw new Error('Method not implemented.');
+  }
+}
+
+export function JsonRpc<TBase extends Constructor<JsonRpcBase>>(Base: TBase) {
+  return class Jsonrpc extends Base {
+    private __methods = new Set(Object.keys(rpcMethods));
+    private __genesises: Set<string>;
+
+    setGenesises(genesises: string[]) {
+      this.__genesises = new Set(genesises);
+    }
+
+    _getMethod(name: string) {
+      if (!this.__methods.has(name)) {
+        throw new MethodNotFound();
+      }
+      return rpcMethods[name];
+    }
+
+    async handleRequest(req: JsonRpcRequest | JsonRpcRequest[]): Promise<JsonRpcResponse | JsonRpcResponse[]> {
+      if (Array.isArray(req)) {
+        return Promise.all(req.map((r) => this.executeMethod(r)));
+      } else {
+        return this.executeMethod(req);
+      }
+    }
+
+    async executeMethod({ method, params, id }: JsonRpcRequest): Promise<JsonRpcResponse> {
+      try {
+        if (!params.genesis) {
+          throw new GenesisNotFound();
+        }
+        if (!this.__genesises.has(params.genesis)) {
+          throw new NetworkNotSupported(params.genesis);
+        }
+
+        const result = await this._getMethod(method).apply(this, [params]);
+        return {
+          jsonrpc: '2.0',
+          id,
+          result,
+        };
+      } catch (err) {
+        return {
+          jsonrpc: '2.0',
+          id,
+          error: {
+            code: err.code || -32603,
+            message: err.message,
+            data: err.data || undefined,
+          },
+        };
+      }
+    }
+  };
+}

idea/explorer/src/decorators/pagination.ts

@@ -0,0 +1,15 @@
+export function Pagination() {
+  return function (target: any, propKey: string, descriptor: PropertyDescriptor) {
+    const originalMethod = descriptor.value;
+
+    descriptor.value = function (...args: any[]) {
+      const [params] = args;
+
+      params.limit = Math.min(params.limit, 100) || 20;
+
+      params.offset = params.offset || 0;
+
+      return originalMethod.apply(this, args);
+    };
+  };
+}

idea/explorer/src/decorators/required.ts

@@ -0,0 +1,19 @@
+import { InvalidParams } from '../errors';
+
+export function RequiredParams(params: string[]) {
+  return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+    const originalMethod = descriptor.value;
+
+    descriptor.value = function (...args: any[]) {
+      const missing = params.filter((param) => args[0][param] === undefined);
+
+      if (missing.length) {
+        throw new InvalidParams(`Missing required params: ${missing.join(', ')}`);
+      }
+
+      return originalMethod.apply(this, args);
+    };
+
+    return descriptor;
+  };
+}

idea/explorer/src/decorators/validation.ts

@@ -0,0 +1,36 @@
+import {
+  registerDecorator,
+  ValidationOptions,
+  ValidatorConstraint,
+  ValidatorConstraintInterface,
+  ValidationArguments,
+  isEmpty,
+} from 'class-validator';
+
+@ValidatorConstraint({ async: false })
+class IsOneOfConstraint implements ValidatorConstraintInterface {
+  validate(value: any, { constraints }: ValidationArguments) {
+    const [allowedValues, required] = constraints;
+    if (!required && isEmpty(value)) {
+      return true;
+    }
+    return allowedValues.includes(value);
+  }
+
+  defaultMessage(args: ValidationArguments) {
+    const [allowedValues] = args.constraints;
+    return `Value must be one of the following: ${allowedValues.join(', ')}`;
+  }
+}
+
+export function IsOneOf(values: string[], required = false, options?: ValidationOptions) {
+  return function (obj: any, propertyName: string) {
+    registerDecorator({
+      target: obj.constructor,
+      propertyName,
+      options,
+      constraints: [values, required],
+      validator: IsOneOfConstraint,
+    });
+  };
+}

idea/explorer/src/errors/base.ts

@@ -0,0 +1,16 @@
+import { JsonRpcError } from '../types';
+
+export class GenesisNotFound implements JsonRpcError {
+  code = -32601;
+  message = 'Genesis not found in the request';
+}
+
+export class NetworkNotSupported implements JsonRpcError {
+  code = -32602;
+  message = 'Network is not supported';
+  data = undefined;
+
+  constructor(public genesis: string) {
+    this.data = 'genesis: ' + genesis;
+  }
+}

idea/explorer/src/errors/code.ts

@@ -0,0 +1,6 @@
+import { JsonRpcError } from '../types';
+
+export class CodeNotFound implements JsonRpcError {
+  code = -32404;
+  message = 'Code not found';
+}

idea/explorer/src/errors/event.ts

@@ -0,0 +1,6 @@
+import { JsonRpcError } from '../types';
+
+export class EventNotFound implements JsonRpcError {
+  code = -32404;
+  message = 'Event not found';
+}

idea/explorer/src/errors/index.ts

@@ -0,0 +1,6 @@
+export * from './code';
+export * from './program';
+export * from './message';
+export * from './event';
+export * from './jsonrpc';
+export * from './base';

idea/explorer/src/errors/jsonrpc.ts

@@ -0,0 +1,30 @@
+import { JsonRpcError } from '../types';
+
+export class MethodNotFound implements JsonRpcError {
+  code = -32601;
+  message = 'Method not found';
+}
+
+export class InvalidParams implements JsonRpcError {
+  code = -32602;
+  message = 'Invalid params';
+  data = undefined;
+
+  constructor(details?: string) {
+    if (details) {
+      this.data = details;
+    }
+  }
+}
+
+export class InternalError implements JsonRpcError {
+  code = -32603;
+  message = 'Internal error';
+  data = undefined;
+
+  constructor(details?: string) {
+    if (details) {
+      this.data = details;
+    }
+  }
+}

idea/explorer/src/errors/message.ts

@@ -0,0 +1,6 @@
+import { JsonRpcError } from '../types';
+
+export class MessageNotFound implements JsonRpcError {
+  code = -32404;
+  message = 'Message not found';
+}