Son/techn 371 fix todo tasks discovery scope

runtime/backend/src/discovery/models/AccountDiscoveryStateData.ts

@@ -24,6 +24,18 @@ import type { StateData } from "../../common/models/StateData";
  * @since v0.1.0
  */
 export class AccountDiscoveryStateData implements StateData {
+  /**
+   * Contains the *address* of the last account that was used
+   * in the *transaction* discovery. This is used in the case
+   * of runtime configuration that contains **more than one**
+   * discovery source and permits to track multiple accounts
+   * sequentially.
+   *
+   * @access public
+   * @var {string}
+   */
+  public lastUsedAccount: string;
+
   /**
    * Contains the last page number that was used in the
    * transactions database query. Since accounts are all

runtime/backend/src/discovery/models/AssetSchema.ts

@@ -27,7 +27,6 @@ import { AssetDTO } from "./AssetDTO";
  * Note that this class uses the generic {@link Transferable} trait to
  * enable a `toDTO()` method on the model.
  *
- * @todo The {@link Asset} model does not need fields to be **public**.
  * @todo Timestamp fields should be **numbers** to avoid timezone issues.
  * @since v0.3.0
  */

runtime/backend/src/discovery/models/BlockSchema.ts

@@ -27,7 +27,6 @@ import { BlockDTO } from "./BlockDTO";
  * Note that this class uses the generic {@link Transferable} trait to
  * enable a `toDTO()` method on the model.
  *
- * @todo The {@link Asset} model does not need fields to be **public**.
  * @todo Timestamp fields should be **numbers** to avoid timezone issues.
  * @since v0.3.2
  */

runtime/backend/src/discovery/schedulers/DiscoverAccounts/DiscoverAccounts.ts

@@ -12,7 +12,6 @@ import { Injectable } from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
 import { InjectModel } from "@nestjs/mongoose";
 import { Cron } from "@nestjs/schedule";
-import { PublicAccount, NetworkType } from "@dhealth/sdk";
 
 // internal dependencies
 import { QueryParameters } from "../../../common/concerns/Queryable";
@@ -41,9 +40,6 @@ import { LogService } from "../../../common/services/LogService";
  * scheduler. Contains source code for the execution logic of a
  * command with name: `discovery:DiscoverAccounts`.
  *
- * @todo This discovery should use a specific **discovery**
- * @todo config field instead of dappPublicKey
- * @todo (similar to getNextSource in DiscoverTransactions)
  * @since v0.1.0
  */
 @Injectable()
@@ -85,6 +81,15 @@ export class DiscoverAccounts extends DiscoveryCommand {
    */
   private usePageSize = 100;
 
+  /**
+   * Memory store for the last account being read. This is used
+   * in {@link getStateData} to update the latest execution state.
+   *
+   * @access private
+   * @var {string}
+   */
+  private lastUsedAccount: string;
+
   /**
    * The constructor of this class.
    * Params will be automatically injected upon called.
@@ -112,6 +117,7 @@ export class DiscoverAccounts extends DiscoveryCommand {
     // sets default state data
     this.lastPageNumber = 1;
     this.lastExecutedAt = new Date().valueOf();
+    this.lastUsedAccount = "";
   }
 
   /**
@@ -165,6 +171,7 @@ export class DiscoverAccounts extends DiscoveryCommand {
    */
   protected getStateData(): AccountDiscoveryStateData {
     return {
+      lastUsedAccount: this.lastUsedAccount,
       lastPageNumber: this.lastPageNumber,
       lastExecutedAt: this.lastExecutedAt,
     } as AccountDiscoveryStateData;
@@ -191,25 +198,23 @@ export class DiscoverAccounts extends DiscoveryCommand {
    */
   @Cron("0 */2 * * * *", { name: "discovery:cronjobs:accounts" })
   public async runAsScheduler(): Promise<void> {
-    // accounts discovery cronjob always read the dApp's main account
-    const dappPubKey = this.configService.get<string>("dappPublicKey");
-    const networkType = this.configService.get<NetworkType>(
-      "network.networkIdentifier",
-    );
+    // CAUTION:
+    // accounts discovery cronjob uses discovery sources
+    const sources = this.configService.get<string[]>("discovery.sources");
 
-    // creates the discovery source public account
-    const publicAcct = PublicAccount.createFromPublicKey(
-      dappPubKey,
-      networkType,
-    );
+    // iterate through all configured discovery sources and
+    // fetch assets from db. Accounts will first be read for
+    // sources that are *not yet synchronized*.
+    const source: string = await this.getNextSource(sources);
+    this.lastUsedAccount = source;
 
     // keep track of last execution
     this.lastExecutedAt = new Date().valueOf();
 
     // executes the actual command logic (this will call discover())
     await super.run([], {
-      source: publicAcct.address.plain(),
-      debug: false,
+      source,
+      debug: true,
     } as DiscoveryCommandOptions);
   }
 
@@ -245,17 +250,20 @@ export class DiscoverAccounts extends DiscoveryCommand {
       this.lastPageNumber = this.state.data.lastPageNumber ?? 1;
     }
 
-    // check for the total number of transactions
-    const transactionsState = await this.stateService.findOne(
-      new StateQuery({
-        name: "discovery:DiscoverTransactions",
-      } as StateDocument),
-    );
+    // per-source synchronization: "discovery:DiscoverTransactions:%SOURCE%"
+    const stateIdentifier = `${this.stateIdentifier}:${options.source}`;
+    const stateQuerySrc = new StateQuery({
+      name: stateIdentifier,
+    } as StateDocument);
+
+    // fetch **per-source** synchronization state once
+    // Caution: do not confuse with `this.state`, this one
+    // is internal and synchronizes **per each source**.
+    const state = await this.stateService.findOne(stateQuerySrc);
 
     const countTransactions =
-      !!transactionsState &&
-      "totalNumberOfTransactions" in transactionsState.data
-        ? transactionsState.data.totalNumberOfTransactions
+      !!state && "totalNumberOfTransactions" in state.data
+        ? state.data.totalNumberOfTransactions
         : 0;
 
     // if we reached the end of transactions, we want
@@ -278,6 +286,7 @@ export class DiscoverAccounts extends DiscoveryCommand {
 
     // (1) each round queries a page of 100 transactions *from the database*
     // and discovers addresses that are involved in said transactions
+    let isSynchronized: boolean;
     for (
       let i = this.lastPageNumber, max = this.lastPageNumber + 10;
       i < max;
@@ -301,6 +310,9 @@ export class DiscoverAccounts extends DiscoveryCommand {
         break;
       }
 
+      // determines the current source's synchronization state
+      isSynchronized = transactions.isLastPage();
+
       // proceeds to extracting accounts from transactions
       this.discoveredAddresses = this.discoveredAddresses
         .concat(
@@ -358,10 +370,14 @@ export class DiscoverAccounts extends DiscoveryCommand {
       this.debugLog(`Skipped ${nSkipped} account(s) that already exist`);
     }
 
-    // a per-command state update is *not* necessary here because
-    // the `BaseCommand` class' `run` method automatically updates
-    // the per-command state with updated values *after* executing
-    // this discovery method.
+    // (4) update per-source state `lastPageNumber`
+    await this.stateService.updateOne(
+      stateQuerySrc, // /!\ per-source
+      {
+        lastPageNumber: this.lastPageNumber,
+        sync: isSynchronized,
+      },
+    );
 
     // no-return (void)
   }

runtime/backend/src/discovery/schedulers/DiscoverAssets/DiscoverAssets.ts

@@ -190,7 +190,6 @@ export class DiscoverAssets extends DiscoveryCommand {
    * <br /><br />
    * This scheduler is registered to run **every 2 minutes**.
    *
-   * @todo This discovery should use a specific **discovery** config field instead of dappPublicKey
    * @see BaseCommand
    * @access public
    * @async

runtime/backend/src/discovery/schedulers/DiscoverTransactions/DiscoverTransactions.ts

@@ -92,7 +92,6 @@ export interface DiscoverTransactionsCommandOptions
  * scheduler. Contains source code for the execution logic of a
  * command with name: `discovery:DiscoverTransactions`.
  *
- * @todo Should use `BigInt` in {@link extractTransactionBlock} because `height.compact()` is not protected for number overflow.
  * @since v0.2.0
  */
 @Injectable()
@@ -332,10 +331,9 @@ export class DiscoverTransactions extends DiscoveryCommand {
 
     // executes the actual command logic (this will call discover())
     // additionally, updates state.data.lastUsedAccount
-    // @todo remove debug flag for staging/production releases
     await this.run(["both"], {
       source,
-      debug: false,
+      debug: process.env.NODE_ENV === "development",
     } as DiscoveryCommandOptions);
   }
 
@@ -736,7 +734,6 @@ export class DiscoverTransactions extends DiscoveryCommand {
    * The transaction header can always be re-created using the other fields
    * present in the {@link Transaction} document.
    *
-   * @todo Move to a bytes-optimized storage format for payloads (only message is necessary)
    * @param {Transaction} transaction
    * @returns {string}
    */

runtime/backend/tests/unit/discovery/schedulers/DiscoverAccounts.spec.ts

@@ -193,21 +193,23 @@ describe("discovery/DiscoverAccounts", () => {
         .mockReturnValue({
           address: { plain: () => "NDAPPH6ZGD4D6LBWFLGFZUT2KQ5OLBLU32K3HNY" }
         } as any);
+      jest
+        .spyOn((service as any), "getNextSource")
+        .mockResolvedValue("test-source");
 
       // act
       await service.runAsScheduler();
 
       // assert
-      expect(configServiceGetCall).toHaveBeenCalledTimes(2);
-      expect(configServiceGetCall).toHaveBeenCalledWith("dappPublicKey");
-      expect(configServiceGetCall).toHaveBeenCalledWith("network.networkIdentifier");
+      expect(configServiceGetCall).toHaveBeenCalledTimes(1);
+      expect(configServiceGetCall).toHaveBeenCalledWith("discovery.sources");
       expect((service as any).lastExecutedAt).toBe(1643673600000);
       expect(superRun).toHaveBeenNthCalledWith(
         1,
         [],
         {
-          source: "NDAPPH6ZGD4D6LBWFLGFZUT2KQ5OLBLU32K3HNY",
-          debug: false,
+          source: "test-source",
+          debug: true,
         }
       );
     });
@@ -258,8 +260,10 @@ describe("discovery/DiscoverAccounts", () => {
       // prepare
       const transactionsServiceFindCall = jest.spyOn(transactionsService, "find").mockResolvedValueOnce({
         data: fakeCreateTransactionDocuments(100), // full page ONCE
+        isLastPage: () => false,
       } as PaginatedResultDTO<TransactionDocument>).mockResolvedValueOnce({
         data: [], // empty page
+        isLastPage: () => true,
       } as PaginatedResultDTO<TransactionDocument>);
 
       // act
@@ -294,8 +298,10 @@ describe("discovery/DiscoverAccounts", () => {
       // prepare
       const transactionsServiceFindCall = jest.spyOn(transactionsService, "find").mockResolvedValueOnce({
         data: fakeCreateTransactionDocuments(100), // full page ONCE
+        isLastPage: () => false,
       } as PaginatedResultDTO<TransactionDocument>).mockResolvedValueOnce({
         data: fakeCreateTransactionDocuments(20), // not full
+        isLastPage: () => false,
       } as PaginatedResultDTO<TransactionDocument>);
 
       // act
@@ -330,6 +336,7 @@ describe("discovery/DiscoverAccounts", () => {
       // prepare
       const transactionsServiceFindCall = jest.spyOn(transactionsService, "find").mockResolvedValueOnce({
         data: fakeCreateTransactionDocuments(100), // full page
+        isLastPage: () => false,
       } as PaginatedResultDTO<TransactionDocument>);
 
       // act
@@ -405,6 +412,7 @@ describe("discovery/DiscoverAccounts", () => {
       .spyOn(transactionsService, "find")
         .mockResolvedValue({
           data: fakeCreateTransactionDocuments(3), // page is not full
+          isLastPage: () => false,
         } as PaginatedResultDTO<TransactionDocument>);
 
       // act
@@ -517,7 +525,8 @@ describe("discovery/DiscoverAccounts", () => {
         .fn()
         .mockResolvedValue({
           data: fakeCreateTransactionDocuments(3), // page is not full
-        });
+          isLastPage: () => false,
+        } as PaginatedResultDTO<TransactionDocument>);
       (service as any).transactionsService.find = transactionsServiceFindCall;
 
       // act