feat(idea/meta-storage): support sails idl

.github/workflows/e2e-tests.yaml

@@ -33,7 +33,7 @@ env:
   REACT_APP_DEFAULT_NODES_URL: ${{ secrets.REACT_APP_DEFAULT_NODES_URL }}
   REACT_APP_RRT: ''
   REACT_APP_HCAPTCHA_SITE_KEY: ${{ secrets.REACT_APP_HCAPTCHA_SITE_KEY }}
-  NIGHTLY_TOOLCHAIN_VERSION: ${{ vars.NIGHTLY_TOOLCHAIN_VERSION }}
+  BINARYEN_VERSION: version_111
 
 jobs:
   check-labels:
@@ -303,51 +303,34 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Install NodeJS 18.x
+      - name: Install NodeJS 20.x
         uses: actions/[email protected]
         with:
-          node-version: 18.x
-
-      - name: "Install dependencies"
-        run: YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install
-
-      - name: "Build: @gear-js/common"
-        run: yarn build:common
-
-      - name: "Install: Nightly toolchain"
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          target: wasm32-unknown-unknown
-          components: llvm-tools-preview
-
-      - name: "Install: Show specific nightly version"
-        if: ${{ env.NIGHTLY_TOOLCHAIN_VERSION != '' }}
-        run: echo $NIGHTLY_TOOLCHAIN_VERSION | sed 's/-/ - /g'
-
-      - name: "Install: Specific nightly toolchain"
-        if: ${{ env.NIGHTLY_TOOLCHAIN_VERSION != '' }}
-        uses: actions-rs/toolchain@v1
+          node-version: 20.x
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-${{ env.NIGHTLY_TOOLCHAIN_VERSION }}
-          target: wasm32-unknown-unknown
-          components: llvm-tools-preview
+          targets: wasm32-unknown-unknown        
 
-      - name: "Install: Pin to specific nightly toolchain"
-        if: ${{ env.NIGHTLY_TOOLCHAIN_VERSION != '' }}
-        run: |
-          rm -rf $RUSTUP_HOME/toolchains/nightly-x86_64-unknown-linux-gnu
-          ln -s $RUSTUP_HOME/toolchains/nightly-$NIGHTLY_TOOLCHAIN_VERSION-x86_64-unknown-linux-gnu $RUSTUP_HOME/toolchains/nightly-x86_64-unknown-linux-gnu
-
       - name: "Prepare: Build test programs"
         working-directory: api/programs
         run: cargo build --release
 
       - name: "Prepare: Copy built programs"
-        run: |
-          cp api/programs/target/wasm32-unknown-unknown/release/* idea/tests/wasm-test
+        run: cp api/programs/target/wasm32-unknown-unknown/release/* idea/tests/wasm-test
+
+      - name: "Prepare: Build test programs"
+        working-directory: idea/tests/programs
+        run: cargo build --release
+
+      - name: "Install dependencies"
+        run: YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install
+
+      - name: "Build: @gear-js/common"
+        run: yarn build:common
 
       - name: "Run tests"
         timeout-minutes: 7

api/programs/rust-toolchain.toml

@@ -1,5 +1,5 @@
 [toolchain]
-channel = "nightly-2023-10-14"
+channel = "nightly-2024-01-25"
 components = [ "llvm-tools" ]
 targets = [ "wasm32-unknown-unknown" ]
 profile = "default"

idea/api-gateway/src/middleware/validate-json-rpc-request.middleware.ts

@@ -17,21 +17,20 @@ export async function validateJsonRpcRequestMiddleware({ body }: Request, res: R
   next();
 }
 
-function isValidRequestParams({ id, method, jsonrpc, params }: IRpcRequest): boolean {
-  return !!id && !!method && !!jsonrpc && !!params;
-}
+const isValidRequestParams = ({ id, method, jsonrpc, params }: IRpcRequest): boolean =>
+  !!id && !!method && !!jsonrpc && !!params;
 
 function getInvalidParamsResponse(req: IRpcRequest) {
   logger.info('Invalid params error', { req });
 
-  const error = JSONRPC_ERRORS.InvalidParams.name;
+  const error = JSONRPC_ERRORS.InvalidParams;
 
   return {
     jsonrpc: '2.0',
     id: req.id || null,
     error: {
-      message: JSONRPC_ERRORS[error].message,
-      code: JSONRPC_ERRORS[error].code,
+      message: error.message,
+      code: error.code,
     },
   };
 }

idea/common/src/enums/api-methods.enum.ts

@@ -25,6 +25,8 @@ export enum INDEXER_INTERNAL_METHODS {
 export enum META_STORAGE_METHODS {
   META_GET = 'meta.get',
   META_ADD = 'meta.add',
+  SAILS_ADD = 'sails.add',
+  SAILS_GET = 'sails.get',
 }
 
 export enum META_STORAGE_INTERNAL_METHODS {

idea/common/src/interfaces/api-response.ts

@@ -38,6 +38,11 @@ interface GetAllStateResult {
 interface ProgramDataResult extends Omit<IProgram, 'meta'> {
   meta?: { meta?: string };
 }
+
+interface AddSailsResult {
+  status: 'Sails idl added';
+}
+
 interface IRpcResponse {
   jsonrpc: '2.0';
   id: number;
@@ -50,7 +55,8 @@ interface IRpcResponse {
     | IMessage[]
     | AddMetaResult
     | GetAllCodeResult
-    | GetStatesResult;
+    | GetStatesResult
+    | AddSailsResult;
   error?: IRpcError;
 }
 
@@ -70,4 +76,5 @@ export {
   IRpcResponse,
   IRpcError,
   GetStatesResult,
+  AddSailsResult,
 };

idea/common/src/interfaces/rpc-request/meta-storage.ts

@@ -13,3 +13,8 @@ export interface GetMetaParams {
   hash: string;
   hex: HexString;
 }
+
+export interface AddSailsIdlParams {
+  codeId: string;
+  data: string;
+}

idea/common/src/jsonrpc-errors.ts

@@ -109,6 +109,11 @@ export const JSONRPC_ERRORS = {
     code: -32400,
     message: 'Address is not supported',
   },
+  SailsIdlNotFound: {
+    name: 'SailsIdlNotFound',
+    code: -32404,
+    message: 'Sails IDL not found',
+  },
 };
 
 export function isExistError(name: string) {

idea/meta-storage/package.json

@@ -17,7 +17,7 @@
   "scripts": {
     "build": "rm -rf dist && tsc",
     "start": "node dist/main.js",
-    "watch": "ts-node-dev src/main.ts"
+    "watch": "clear && ts-node-dev src/main.ts"
   },
   "lint-staged": {
     "*.ts": [

idea/meta-storage/src/database/data-source.ts

@@ -1,6 +1,6 @@
 import { DataSource } from 'typeorm';
 
-import { Meta } from './entities';
+import { Code, Meta, SailsIdl } from './entities';
 import config from '../config';
 
 export const AppDataSource = new DataSource({
@@ -10,7 +10,7 @@ export const AppDataSource = new DataSource({
   username: config.db.user,
   password: config.db.password,
   database: config.db.name,
-  entities: [Meta],
+  entities: [Meta, SailsIdl, Code],
   synchronize: true,
   logging: ['error', 'schema'],
 });

idea/meta-storage/src/database/entities/code.entity.ts

@@ -0,0 +1,15 @@
+import { Entity, ManyToOne, PrimaryColumn } from 'typeorm';
+import { SailsIdl } from './sails.entity';
+
+@Entity()
+export class Code {
+  constructor(props: Partial<Code>) {
+    Object.assign(this, props);
+  }
+
+  @PrimaryColumn()
+  public id: string;
+
+  @ManyToOne(() => SailsIdl, (sails) => sails.id)
+  public sails: SailsIdl;
+}

idea/meta-storage/src/database/entities/index.ts

@@ -1 +1,3 @@
 export { Meta } from './meta.entity';
+export { SailsIdl } from './sails.entity';
+export { Code } from './code.entity';

idea/meta-storage/src/database/entities/sails.entity.ts

@@ -0,0 +1,18 @@
+import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm';
+import { Code } from './code.entity';
+
+@Entity()
+export class SailsIdl {
+  constructor(props: Partial<SailsIdl>) {
+    Object.assign(this, props);
+  }
+
+  @PrimaryColumn()
+  public id: string;
+
+  @Column()
+  public data: string;
+
+  @OneToMany(() => Code, (code) => code.id)
+  public codes: Code[];
+}

idea/meta-storage/src/rmq.ts

@@ -23,6 +23,8 @@ export class RMQService {
     this.methods = {
       [META_STORAGE_METHODS.META_ADD]: this.metaService.addMetaDetails.bind(this.metaService),
       [META_STORAGE_METHODS.META_GET]: this.metaService.get.bind(this.metaService),
+      [META_STORAGE_METHODS.SAILS_ADD]: this.metaService.addIdl.bind(this.metaService),
+      [META_STORAGE_METHODS.SAILS_GET]: this.metaService.getIdl.bind(this.metaService),
       [META_STORAGE_INTERNAL_METHODS.META_HASH_ADD]: this.metaService.addMeta.bind(this.metaService),
     };
   }
@@ -81,9 +83,13 @@ export class RMQService {
           const params = JSON.parse(msg.content.toString());
           const correlationId = msg.properties.correlationId;
 
-          const result = await this.handleIncomingMsg(method, params);
+          try {
+            const result = await this.handleIncomingMsg(method, params);
 
-          this.sendMsg(RMQExchange.DIRECT_EX, RMQQueue.REPLIES, result, correlationId);
+            this.sendMsg(RMQExchange.DIRECT_EX, RMQQueue.REPLIES, result, correlationId);
+          } catch (error) {
+            logger.error('Failed to handle incoming message', { error: error.message, stack: error.stack });
+          }
         },
         { noAck: true },
       );

idea/meta-storage/src/service.ts

@@ -1,16 +1,24 @@
-import { Repository } from 'typeorm';
 import { AddMetaDetailsParams, AddMetahashParams, GetMetaParams, logger } from '@gear-js/common';
+import { ProgramMetadata, MetadataVersion, HumanTypesRepr } from '@gear-js/api';
+import { Repository } from 'typeorm';
+import * as crypto from 'crypto';
 
-import { Meta, AppDataSource } from './database';
-import { InvalidParamsError, MetaNotFoundError } from './util/errors';
+import { Meta, AppDataSource, SailsIdl } from './database';
+import { InvalidParamsError, MetaNotFoundError, SailsIdlNotFoundError } from './util/errors';
 import { validateMetaHex } from './util/validate';
-import { ProgramMetadata, MetadataVersion, HumanTypesRepr } from '@gear-js/api';
+import { Code } from './database/entities/code.entity';
+
+const getHash = (data: string) => crypto.createHash('sha256').update(data).digest('hex');
 
 export class MetaService {
   private metaRepo: Repository<Meta>;
+  private sailsRepo: Repository<SailsIdl>;
+  private codeRepo: Repository<Code>;
 
   constructor() {
     this.metaRepo = AppDataSource.getRepository(Meta);
+    this.sailsRepo = AppDataSource.getRepository(SailsIdl);
+    this.codeRepo = AppDataSource.getRepository(Code);
   }
 
   async addMeta({ metahash }: AddMetahashParams): Promise<string[]> {
@@ -82,4 +90,41 @@ export class MetaService {
     const meta = await this.metaRepo.find({ where: { hasState: true }, select: { hash: true } });
     return meta.map((m) => m.hash);
   }
+
+  async addIdl({ codeId, data }) {
+    if (!codeId || !data) {
+      throw new InvalidParamsError();
+    }
+
+    const hash = getHash(data);
+
+    logger.info('Adding IDL', { codeId, hash });
+
+    let sails = await this.sailsRepo.findOne({ where: { id: hash } });
+
+    if (!sails) {
+      const code = await this.codeRepo.findOne({ where: { id: codeId } });
+      if (code) {
+        throw new InvalidParamsError('Code already has IDL');
+      }
+      sails = new SailsIdl({ id: hash, data });
+    }
+
+    const code = new Code({ id: codeId, sails });
+
+    await this.sailsRepo.save(sails);
+    await this.codeRepo.save(code);
+
+    return { status: 'Sails idl added' };
+  }
+
+  async getIdl({ codeId }) {
+    const code = await this.codeRepo.findOne({ where: { id: codeId }, relations: { sails: true } });
+
+    if (!code) {
+      throw new SailsIdlNotFoundError();
+    }
+
+    return code.sails.data;
+  }
 }

idea/meta-storage/src/util/errors.ts

@@ -11,3 +11,7 @@ export class InvalidMetadataError extends Error {
 export class InvalidParamsError extends Error {
   name = JSONRPC_ERRORS.InvalidParams.name;
 }
+
+export class SailsIdlNotFoundError extends Error {
+  name = JSONRPC_ERRORS.SailsIdlNotFound.name;
+}

idea/tests/.gitignore

@@ -1,7 +1,9 @@
-wasm/
+./wasm/
 .env
 dist/
 .yarn/
 wasm-test/*
 !wasm-test/app.opt.wasm
 genesis
+
+target/

idea/tests/e2e/meta-storage.test.ts

@@ -5,6 +5,12 @@ import * as path from 'path';
 import { PATH_TO_PROGRAMS } from './config';
 import { generateCodeHash, HexString } from '@gear-js/api';
 
+const PING_WASM_PATH = './programs/target/wasm32-unknown-unknown/release/ping.opt.wasm';
+const PING_IDL_PATH = './programs/ping-sails/wasm/ping.idl';
+
+const pingCode = fs.readFileSync(PING_WASM_PATH);
+const pingIdl = fs.readFileSync(PING_IDL_PATH, 'utf-8');
+
 const meta: HexString = `0x${fs.readFileSync(path.join(PATH_TO_PROGRAMS, 'test_meta.meta.txt'), 'utf-8')}`;
 const hash = generateCodeHash(meta);
 
@@ -24,4 +30,20 @@ describe('meta-storage methods', () => {
     expect(response.result).toHaveProperty('hash', hash);
     expect(response.result).toHaveProperty('hex', meta);
   });
+
+  test(META_STORAGE_METHODS.SAILS_ADD, async () => {
+    const codehash = generateCodeHash(pingCode);
+    const response = await request(META_STORAGE_METHODS.SAILS_ADD, { codeId: codehash, data: pingIdl });
+
+    expect(response).toHaveProperty('result');
+    expect(response.result).toHaveProperty('status', 'Sails idl added');
+  });
+
+  test(META_STORAGE_METHODS.SAILS_GET, async () => {
+    const codehash = generateCodeHash(pingCode);
+    const response = await request(META_STORAGE_METHODS.SAILS_GET, { codeId: codehash });
+
+    expect(response).toHaveProperty('result');
+    expect(response.result).toBe(pingIdl);
+  });
 });