reimburse attempt2

Signed-off-by: microwavedcola1 <[email protected]>

src/scripts/assets/output.csv

@@ -0,0 +1,5 @@
+index,owner,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+0,9Ut1gZJnd5D7EjPXm2DygYWZkZGpt5QSMEYAaVx2hur4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
+1,9Ut1gZJnd5D7EjPXm2DygYWZkZGpt5QSMEYAaVx2hur4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
+2,9Ut1gZJnd5D7EjPXm2DygYWZkZGpt5QSMEYAaVx2hur4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
+3,9Ut1gZJnd5D7EjPXm2DygYWZkZGpt5QSMEYAaVx2hur4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1

src/scripts/reimburse.ts

@@ -7,18 +7,25 @@ import * as path from 'path';
 import * as csv from 'fast-csv';
 import {
   Commitment,
+  ComputeBudgetProgram,
   Connection,
   Keypair,
   PublicKey,
-  sendAndConfirmTransaction,
   SystemProgram,
   Transaction,
   TransactionInstruction,
 } from '@solana/web3.js';
 import BN from 'bn.js';
 import { MangoClient } from '../client';
 import { Cluster, Config } from '../config';
-import fs from 'fs';
+import fs, { stat } from 'fs';
+import _ from 'lodash';
+import { sleep } from '@blockworks-foundation/mango-client';
+import { bs58 } from '@project-serum/anchor/dist/cjs/utils/bytes';
+
+const SEND_TRANSACTION_INTERVAL_MS = 10;
+const TRANSACTION_RESEND_INTERVAL_S = 4;
+const MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS = 256;
 
 const PAYER_KEYPAIR = process.env.MB_PAYER_KEYPAIR;
 const PAYER = Keypair.fromSecretKey(
@@ -67,24 +74,10 @@ export async function createAssociatedTokenAccountIdempotentInstruction(
   });
 }
 
-async function tokenTransfer(
-  mangoAccountOwnerPk: PublicKey,
-  mint: PublicKey,
-  nativeTokenAmountToReimburse: BN,
+async function didTransferAlreadyHappen(
+  sourceAta: PublicKey,
+  destinationAta: PublicKey,
 ) {
-  const sourceAta = await Token.getAssociatedTokenAddress(
-    ASSOCIATED_TOKEN_PROGRAM_ID,
-    TOKEN_PROGRAM_ID,
-    mint,
-    SOURCE,
-  );
-  const destinationAta = await Token.getAssociatedTokenAddress(
-    ASSOCIATED_TOKEN_PROGRAM_ID,
-    TOKEN_PROGRAM_ID,
-    mint,
-    mangoAccountOwnerPk,
-  );
-
   // Verify that this tx has not happend in last 100 txs for the destinationAta
   const sigs = await connection.getConfirmedSignaturesForAddress2(
     destinationAta,
@@ -101,22 +94,37 @@ async function tokenTransfer(
         accountKey.pubkey.equals(sourceAta),
       )
     ) {
-      console.log(` - already transferred`);
-      return;
+      return true;
     }
   }
+  return false;
+}
+
+async function buildTokenTransferIxs(
+  mangoAccountOwnerPk: PublicKey,
+  mint: PublicKey,
+  nativeTokenAmountToReimburse: BN,
+) {
+  const sourceAta = await Token.getAssociatedTokenAddress(
+    ASSOCIATED_TOKEN_PROGRAM_ID,
+    TOKEN_PROGRAM_ID,
+    mint,
+    SOURCE,
+  );
+  const destinationAta = await Token.getAssociatedTokenAddress(
+    ASSOCIATED_TOKEN_PROGRAM_ID,
+    TOKEN_PROGRAM_ID,
+    mint,
+    mangoAccountOwnerPk,
+  );
 
-  // Build tx
-  const tx = new Transaction();
-  tx.add(
+  return [
     await createAssociatedTokenAccountIdempotentInstruction(
       PAYER.publicKey,
       mangoAccountOwnerPk,
       mint,
       destinationAta,
     ),
-  );
-  tx.add(
     await Token.createTransferInstruction(
       TOKEN_PROGRAM_ID,
       sourceAta,
@@ -128,21 +136,15 @@ async function tokenTransfer(
       ),
       PAYER.publicKey,
       [PAYER],
-      nativeTokenAmountToReimburse.toNumber(), // throws `Note: Blob.encode[amount] requires (length 8) Buffer as src` when BN is used
+      nativeTokenAmountToReimburse.toNumber(),
     ),
-  );
-
-  // Send and confirm
-  const sig = await sendAndConfirmTransaction(connection, tx, [PAYER], {
-    skipPreflight: true,
-  });
-  console.log(` - transferred, sig https://explorer.solana.com/tx/${sig}`);
+  ];
 }
 
 async function reimburseUser(
   mangoAccountOwnerPk: PublicKey,
   nativeTokenAmountsToReimburse: BN[],
-): Promise<void> {
+): Promise<TransactionInstruction[]> {
   const group = await client.getMangoGroup(mangoGroupKey);
   const allTokens = 16;
 
@@ -153,63 +155,195 @@ async function reimburseUser(
     );
   }
 
-  group.tokens.map(async (token, tokenIndex) => {
-    const tokenConfig = groupIds?.tokens.find((tokenConfig) =>
-      token.mint.equals(tokenConfig.mintKey),
-    );
+  return (
+    await Promise.all(
+      group.tokens.map(async (token, tokenIndex) => {
+        const tokenConfig = groupIds?.tokens.find((tokenConfig) =>
+          token.mint.equals(tokenConfig.mintKey),
+        );
 
-    // Token slot empty
-    if (!tokenConfig) {
-      return;
-    }
+        // Token slot empty
+        if (!tokenConfig) {
+          return [];
+        }
 
-    // Token is deactivated
-    if (token.oracleInactive) {
-      return;
-    }
+        // Token is deactivated
+        if (token.oracleInactive) {
+          return [];
+        }
 
-    // Skip if no reimbursements for mint
-    const nativeTokenAmountToReimburse =
-      nativeTokenAmountsToReimburse[tokenIndex];
-    if (nativeTokenAmountToReimburse.eq(new BN(0))) {
-      return;
-    }
+        // Skip if no reimbursements for mint
+        const nativeTokenAmountToReimburse =
+          nativeTokenAmountsToReimburse[tokenIndex];
+        if (nativeTokenAmountToReimburse.eq(new BN(0))) {
+          return [];
+        }
 
-    console.log(
-      `Transferring ${nativeTokenAmountToReimburse} native ${tokenConfig.symbol} (mint - ${tokenConfig.mintKey}) to ${mangoAccountOwnerPk}`,
-    );
+        // console.log(
+        //   `Transferring ${nativeTokenAmountToReimburse} native ${tokenConfig.symbol} (mint - ${tokenConfig.mintKey}) to ${mangoAccountOwnerPk}`,
+        // );
 
-    return await tokenTransfer(
-      mangoAccountOwnerPk,
-      token.mint,
-      nativeTokenAmountToReimburse,
-    );
-  });
+        return await buildTokenTransferIxs(
+          mangoAccountOwnerPk,
+          token.mint,
+          nativeTokenAmountToReimburse,
+        );
+      }),
+    )
+  ).flatMap((res) => res);
+}
+
+async function main() {
+  const rows: {
+    owner: string;
+    0: number;
+    1: number;
+    2: number;
+    3: number;
+    4: number;
+    5: number;
+    6: number;
+    7: number;
+    8: number;
+    9: number;
+    10: number;
+    11: number;
+    12: number;
+    13: number;
+    14: number;
+    15: number;
+  }[] = [];
+
+  fs.createReadStream(path.resolve(__dirname, 'assets', 'output.csv'))
+    .pipe(csv.parse({ headers: true }))
+    .on('data', (row) => rows.push(row))
+    .on('end', async (rowCount: number) => {
+      if (rowCount != rows.length) {
+        throw new Error('Error in aggregating all rows from the csv!');
+      }
+
+      for (const rowChunk of _.chunk(rows, 2)) {
+        const latestBlockhash = await connection.getLatestBlockhash();
+        const txContexts: { owners: string[]; tx: Transaction }[] =
+          await Promise.all(
+            _.chunk(rowChunk, 1).map(async (rowChunksChunk) => {
+              const ixs = (
+                await Promise.all(
+                  rowChunksChunk.map(async (row) => {
+                    return reimburseUser(new PublicKey(row.owner), [
+                      new BN(row['0']),
+                      new BN(row['1']),
+                      new BN(row['2']),
+                      new BN(row['3']),
+                      new BN(row['4']),
+                      new BN(row['5']),
+                      new BN(row['6']),
+                      new BN(row['7']),
+                      new BN(row['8']),
+                      new BN(row['9']),
+                      new BN(row['10']),
+                      new BN(row['11']),
+                      new BN(row['12']),
+                      new BN(row['13']),
+                      new BN(row['14']),
+                      new BN(row['15']),
+                    ]);
+                  }),
+                )
+              ).flatMap((res) => res);
+
+              const tx = new Transaction({
+                blockhash: latestBlockhash.blockhash,
+                lastValidBlockHeight: latestBlockhash.lastValidBlockHeight,
+              });
+              tx.add(
+                ComputeBudgetProgram.requestUnits({
+                  additionalFee: 5000,
+                  units: 1.4e6,
+                }),
+              );
+              tx.add(...ixs);
+
+              return {
+                owners: rowChunk.map((row) => row.owner),
+                tx: tx,
+                confirmed: false,
+              };
+            }),
+          );
+
+        let expiredBlockhashRetries = 5;
+        let blockHeight = await connection.getBlockHeight();
+        while (expiredBlockhashRetries > 0) {
+          const res = await connection.getLatestBlockhash();
+          const lastValidBlockHeight = res.lastValidBlockHeight;
+
+          const pendingTxs = new Map();
+          for (const txContext of txContexts) {
+            txContext.tx.sign(PAYER);
+            if (!txContext.tx.signature) {
+              throw new Error('Tx signature cannott be undefined!');
+            }
+            pendingTxs.set(bs58.encode(txContext.tx.signature), txContext);
+          }
+
+          let lastResend = Date.now() / 1000 - TRANSACTION_RESEND_INTERVAL_S;
+          while (blockHeight <= lastValidBlockHeight) {
+            // Periodically re-send all pending transactions
+            if (
+              Date.now() / 1000 - lastResend >=
+              TRANSACTION_RESEND_INTERVAL_S
+            ) {
+              for (const pendingTxContext of Array.from(pendingTxs.values())) {
+                await connection.sendRawTransaction(
+                  pendingTxContext.tx.serialize(),
+                );
+                // Maintain 100 TPS
+                await sleep(SEND_TRANSACTION_INTERVAL_MS);
+              }
+              lastResend = Date.now() / 1000;
+            }
+
+            // Wait for the next block before checking for transaction statuses
+            let blockHeightRefreshes = 10;
+            let newBlockHeight = blockHeight;
+            while (blockHeight == newBlockHeight && blockHeightRefreshes > 0) {
+              await sleep(500);
+              newBlockHeight = await connection.getBlockHeight();
+              blockHeightRefreshes -= 1;
+            }
+            blockHeight = newBlockHeight;
+
+            // Collect statuses for the transactions, drop those that are confirmed
+            for (const pendingTxsChunk of _.chunk(
+              Array.from(pendingTxs.keys()),
+              MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
+            )) {
+              console.log(`pendingTxsChunk ${pendingTxsChunk}`);
+              const statuses = await connection.getSignatureStatuses(
+                pendingTxsChunk,
+              );
+              console.log(statuses);
+              statuses.value.forEach((status, i) => {
+                if (
+                  status?.confirmationStatus &&
+                  status.confirmationStatus === 'confirmed'
+                ) {
+                  console.log(
+                    `${pendingTxsChunk[
+                      i
+                    ].toString()} confirmed in ${blockHeight}, with ${
+                      status.confirmationStatus
+                    }`,
+                  );
+                  pendingTxs.delete(pendingTxsChunk[i]);
+                }
+              });
+            }
+          }
+        }
+      }
+    });
 }
 
-// Example
-reimburseUser(new PublicKey('9Ut1gZJnd5D7EjPXm2DygYWZkZGpt5QSMEYAaVx2hur4'), [
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(0),
-  new BN(1), // USDC
-]);
-
-// TODO read csv, grab mango accounts owner, grab token deposits per token, call reimburseUser
-fs.createReadStream(path.resolve(__dirname, 'assets', 'output.csv'))
-  .pipe(csv.parse({ headers: true }))
-  .on('error', (error) => console.error(error))
-  .on('data', (row) => console.log(row.mango_account, row.equity))
-  .on('end', (rowCount: number) => console.log(`Parsed ${rowCount} rows`));
+main();