diff --git a/migrations/1715844926068_add-replacing-txid.js b/migrations/1715844926068_add-replacing-txid.js new file mode 100644 index 000000000..9c490b68f --- /dev/null +++ b/migrations/1715844926068_add-replacing-txid.js @@ -0,0 +1,12 @@ +/* eslint-disable camelcase */ +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ + +exports.up = pgm => { + pgm.addColumn('mempool_txs', { + replaced_by_tx_id: { + type: 'bytea', + } + }); +}; + + diff --git a/migrations/1747760018771_txs-sender-nonce-indexes.js b/migrations/1747760018771_txs-sender-nonce-indexes.js new file mode 100644 index 000000000..7f850a39f --- /dev/null +++ b/migrations/1747760018771_txs-sender-nonce-indexes.js @@ -0,0 +1,21 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = pgm => { + pgm.dropIndex('txs', ['sender_address']); + pgm.createIndex('txs', ['sender_address', 'nonce'], { + where: 'canonical = true AND microblock_canonical = true', + }); + pgm.dropIndex('txs', ['sponsor_address']); + pgm.createIndex('txs', ['sponsor_address', 'nonce'], { + where: 'sponsor_address IS NOT NULL AND canonical = true AND microblock_canonical = true', + }); +}; + +exports.down = pgm => { + pgm.dropIndex('txs', ['sender_address', 'nonce']); + pgm.createIndex('txs', ['sender_address']); + pgm.dropIndex('txs', ['sponsor_address', 'nonce']); + pgm.createIndex('txs', ['sponsor_address']); +}; diff --git a/src/api/controllers/db-controller.ts b/src/api/controllers/db-controller.ts index 46baa65a7..92448aaf8 100644 --- a/src/api/controllers/db-controller.ts +++ b/src/api/controllers/db-controller.ts @@ -1147,6 +1147,7 @@ function parseDbAbstractMempoolTx( const abstractMempoolTx: AbstractMempoolTransaction = { ...baseTx, tx_status: getTxStatusString(dbMempoolTx.status) as MempoolTransactionStatus, + replaced_by_tx_id: dbMempoolTx.replaced_by_tx_id ?? null, receipt_time: dbMempoolTx.receipt_time, receipt_time_iso: unixEpochToIso(dbMempoolTx.receipt_time), }; diff --git a/src/api/schemas/entities/transactions.ts b/src/api/schemas/entities/transactions.ts index 1119efea2..2491e82c7 100644 --- a/src/api/schemas/entities/transactions.ts +++ b/src/api/schemas/entities/transactions.ts @@ -429,6 +429,11 @@ export const AbstractMempoolTransactionProperties = { description: 'Status of the transaction', } ), + replaced_by_tx_id: Nullable( + Type.String({ + description: 'ID of another transaction which replaced this one', + }) + ), receipt_time: Type.Integer({ description: 'A unix timestamp (in seconds) indicating when the transaction broadcast was received by the node.', diff --git a/src/datastore/common.ts b/src/datastore/common.ts index bbee8a601..902916a42 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -306,6 +306,8 @@ export interface DbMempoolFeePriority { export interface DbMempoolTx extends BaseTx { pruned: boolean; + replaced_by_tx_id?: string; + receipt_time: number; post_conditions: string; @@ -897,6 +899,7 @@ export interface MempoolTxQueryResult { type_id: number; anchor_mode: number; status: number; + replaced_by_tx_id?: string; receipt_time: number; receipt_block_height: number; @@ -1242,6 +1245,7 @@ export interface MempoolTxInsertValues { type_id: DbTxTypeId; anchor_mode: DbTxAnchorMode; status: DbTxStatus; + replaced_by_tx_id: PgBytea | null; receipt_time: number; receipt_block_height: number; post_conditions: PgBytea; diff --git a/src/datastore/helpers.ts b/src/datastore/helpers.ts index c54359dce..6965e4a9b 100644 --- a/src/datastore/helpers.ts +++ b/src/datastore/helpers.ts @@ -133,6 +133,7 @@ export const MEMPOOL_TX_COLUMNS = [ 'type_id', 'anchor_mode', 'status', + 'replaced_by_tx_id', 'receipt_time', 'receipt_block_height', 'post_conditions', @@ -302,6 +303,7 @@ export function parseMempoolTxQueryResult(result: MempoolTxQueryResult): DbMempo type_id: result.type_id as DbTxTypeId, anchor_mode: result.anchor_mode as DbTxAnchorMode, status: result.status, + replaced_by_tx_id: result.replaced_by_tx_id, receipt_time: result.receipt_time, post_conditions: result.post_conditions, fee_rate: BigInt(result.fee_rate), @@ -1283,6 +1285,7 @@ export function convertTxQueryResultToDbMempoolTx(txs: TxQueryResult[]): DbMempo ? BigInt(tx.token_transfer_amount) : tx.token_transfer_amount, sponsor_address: tx.sponsor_address ?? undefined, + status: DbTxStatus.Pending, }); dbMempoolTxs.push(dbMempoolTx); } diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index f477b0f05..c54280469 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -114,7 +114,13 @@ class MicroblockGapError extends Error { } } -type TransactionHeader = { txId: string; sender: string; nonce: number }; +type TransactionHeader = { + txId: string; + sender_address: string; + sponsor_address?: string; + sponsored: boolean; + nonce: number; +}; /** * Extends `PgStore` to provide data insertion functions. These added features are usually called by @@ -214,7 +220,9 @@ export class PgWriteStore extends PgStore { } else { const prunableTxs: TransactionHeader[] = data.txs.map(d => ({ txId: d.tx.tx_id, - sender: d.tx.sender_address, + sender_address: d.tx.sender_address, + sponsor_address: d.tx.sponsor_address, + sponsored: d.tx.sponsored, nonce: d.tx.nonce, })); await this.pruneMempoolTxs(sql, prunableTxs); @@ -254,7 +262,9 @@ export class PgWriteStore extends PgStore { sql, orphanedAndMissingTxs.map(tx => ({ txId: tx.tx_id, - sender: tx.sender_address, + sender_address: tx.sender_address, + sponsor_address: tx.sponsor_address, + sponsored: tx.sponsored, nonce: tx.nonce, })) ); @@ -324,6 +334,11 @@ export class PgWriteStore extends PgStore { await q.done(); } + await this.updateReplacedByFeeStatusForTxIds( + sql, + data.txs.map(t => t.tx.tx_id), + false + ); if (!this.isEventReplay) { this.debounceMempoolStat(); } @@ -707,7 +722,9 @@ export class PgWriteStore extends PgStore { sql, microOrphanedTxs.map(tx => ({ txId: tx.tx_id, - sender: tx.sender_address, + sender_address: tx.sender_address, + sponsor_address: tx.sponsor_address, + sponsored: tx.sponsored, nonce: tx.nonce, })) ); @@ -718,7 +735,9 @@ export class PgWriteStore extends PgStore { const prunableTxs: TransactionHeader[] = data.txs.map(d => ({ txId: d.tx.tx_id, - sender: d.tx.sender_address, + sender_address: d.tx.sender_address, + sponsor_address: d.tx.sponsor_address, + sponsored: d.tx.sponsored, nonce: d.tx.nonce, })); const removedTxsResult = await this.pruneMempoolTxs(sql, prunableTxs); @@ -1924,6 +1943,7 @@ export class PgWriteStore extends PgStore { type_id: tx.type_id, anchor_mode: tx.anchor_mode, status: tx.status, + replaced_by_tx_id: tx.replaced_by_tx_id ?? null, receipt_time: tx.receipt_time, receipt_block_height: chainTip.block_height, post_conditions: tx.post_conditions, @@ -1958,11 +1978,12 @@ export class PgWriteStore extends PgStore { tenure_change_pubkey_hash: tx.tenure_change_pubkey_hash ?? null, })); - // Revive mempool txs that were previously dropped + // Revive mempool txs that were previously dropped. const revivedTxs = await sql<{ tx_id: string }[]>` UPDATE mempool_txs SET pruned = false, status = ${DbTxStatus.Pending}, + replaced_by_tx_id = NULL, receipt_block_height = ${values[0].receipt_block_height}, receipt_time = ${values[0].receipt_time} WHERE tx_id IN ${sql(values.map(v => v.tx_id))} @@ -1978,7 +1999,8 @@ export class PgWriteStore extends PgStore { `; txIds.push(...revivedTxs.map(r => r.tx_id)); - const result = await sql<{ tx_id: string }[]>` + // Insert new mempool txs. + const inserted = await sql<{ tx_id: string }[]>` WITH inserted AS ( INSERT INTO mempool_txs ${sql(values)} ON CONFLICT ON CONSTRAINT unique_tx_id DO NOTHING @@ -1993,9 +2015,10 @@ export class PgWriteStore extends PgStore { ) SELECT tx_id FROM inserted `; - txIds.push(...result.map(r => r.tx_id)); - // The incoming mempool transactions might have already been settled - // We need to mark them as pruned to avoid inconsistent tx state + txIds.push(...inserted.map(r => r.tx_id)); + + // The incoming mempool transactions might have already been mined. We need to mark them as + // pruned to avoid inconsistent tx state. const pruned_tx = await sql<{ tx_id: string }[]>` SELECT tx_id FROM txs @@ -2004,7 +2027,7 @@ export class PgWriteStore extends PgStore { canonical = true AND microblock_canonical = true`; if (pruned_tx.length > 0) { - await sql<{ tx_id: string }[]>` + await sql` WITH pruned AS ( UPDATE mempool_txs SET pruned = true @@ -2012,18 +2035,90 @@ export class PgWriteStore extends PgStore { tx_id IN ${sql(pruned_tx.map(t => t.tx_id))} AND pruned = false RETURNING tx_id - ), - count_update AS ( - UPDATE chain_tip SET - mempool_tx_count = mempool_tx_count - (SELECT COUNT(*) FROM pruned), - mempool_updated_at = NOW() ) - SELECT tx_id FROM pruned`; + UPDATE chain_tip SET + mempool_tx_count = mempool_tx_count - (SELECT COUNT(*) FROM pruned), + mempool_updated_at = NOW() + `; } } + await this.updateReplacedByFeeStatusForTxIds(sql, txIds); return txIds; } + /** + * Newly confirmed/pruned/restored transactions may have changed the RBF situation for + * transactions with equal nonces. Look for these cases and update txs accordingly. + * @param sql - SQL client + * @param txIds - Updated mempool tx ids + * @param mempool - If we should look in the mempool for these txs + */ + private async updateReplacedByFeeStatusForTxIds( + sql: PgSqlClient, + txIds: string[], + mempool: boolean = true + ): Promise { + for (const txId of txIds) { + // If a transaction with equal nonce was confirmed in a block, mark all conflicting mempool + // txs as RBF. Otherwise, look for the one with the highest fee in the mempool and RBF all the + // others. + // + // Note that we're not filtering by `pruned` when we look at the mempool, because we want the + // RBF data to be retroactively applied to all conflicting txs we've ever seen. + await sql` + WITH source_tx AS ( + SELECT + (CASE sponsored WHEN true THEN sponsor_address ELSE sender_address END) AS address, + nonce, fee_rate + FROM ${mempool ? sql`mempool_txs` : sql`txs`} + WHERE tx_id = ${txId} + LIMIT 1 + ), + same_nonce_mempool_txs AS ( + SELECT tx_id, fee_rate, receipt_time + FROM mempool_txs + WHERE (sponsor_address = (SELECT address FROM source_tx) OR sender_address = (SELECT address FROM source_tx)) + AND nonce = (SELECT nonce FROM source_tx) + ), + mined_tx AS ( + SELECT tx_id + FROM txs + WHERE (sponsor_address = (SELECT address FROM source_tx) OR sender_address = (SELECT address FROM source_tx)) + AND nonce = (SELECT nonce FROM source_tx) + AND canonical = true + AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC + LIMIT 1 + ), + highest_fee_mempool_tx AS ( + SELECT tx_id + FROM same_nonce_mempool_txs + ORDER BY fee_rate DESC, receipt_time DESC + LIMIT 1 + ), + winning_tx AS ( + SELECT COALESCE((SELECT tx_id FROM mined_tx), (SELECT tx_id FROM highest_fee_mempool_tx)) AS tx_id + ), + txs_to_prune AS ( + SELECT tx_id, pruned + FROM mempool_txs + WHERE tx_id IN (SELECT tx_id FROM same_nonce_mempool_txs) + AND tx_id <> (SELECT tx_id FROM winning_tx) + ), + mempool_count_updates AS ( + UPDATE chain_tip SET + mempool_tx_count = mempool_tx_count - (SELECT COUNT(*) FROM txs_to_prune WHERE pruned = false), + mempool_updated_at = NOW() + ) + UPDATE mempool_txs + SET pruned = TRUE, + status = ${DbTxStatus.DroppedReplaceByFee}, + replaced_by_tx_id = (SELECT tx_id FROM winning_tx) + WHERE tx_id IN (SELECT tx_id FROM txs_to_prune) + `; + } + } + private _debounceMempoolStat: { triggeredAt?: number | null; debounce?: NodeJS.Timeout | null; @@ -2095,12 +2190,20 @@ export class PgWriteStore extends PgStore { } } - async dropMempoolTxs({ status, txIds }: { status: DbTxStatus; txIds: string[] }): Promise { + async dropMempoolTxs({ + status, + txIds, + new_tx_id, + }: { + status: DbTxStatus; + txIds: string[]; + new_tx_id: string | null; + }): Promise { for (const batch of batchIterate(txIds, INSERT_BATCH_SIZE)) { const updateResults = await this.sql<{ tx_id: string }[]>` WITH pruned AS ( UPDATE mempool_txs - SET pruned = TRUE, status = ${status} + SET pruned = TRUE, status = ${status}, replaced_by_tx_id = ${new_tx_id} WHERE tx_id IN ${this.sql(batch)} AND pruned = FALSE RETURNING tx_id ), @@ -2511,7 +2614,13 @@ export class PgWriteStore extends PgStore { const updatedMbTxs = updatedMbTxsQuery.map(r => parseTxQueryResult(r)); const txsToPrune: TransactionHeader[] = updatedMbTxs .filter(tx => tx.canonical && tx.microblock_canonical) - .map(tx => ({ txId: tx.tx_id, sender: tx.sender_address, nonce: tx.nonce })); + .map(tx => ({ + txId: tx.tx_id, + sender_address: tx.sender_address, + sponsor_address: tx.sponsor_address, + sponsored: tx.sponsored, + nonce: tx.nonce, + })); const removedTxsResult = await this.pruneMempoolTxs(sql, txsToPrune); if (removedTxsResult.removedTxs.length > 0) { logger.debug( @@ -2681,23 +2790,38 @@ export class PgWriteStore extends PgStore { if (transactions.length === 0) return { restoredTxs: [] }; if (logger.isLevelEnabled('debug')) for (const tx of transactions) - logger.debug(`Restoring mempool tx: ${tx.txId} sender: ${tx.sender} nonce: ${tx.nonce}`); + logger.debug( + `Restoring mempool tx: ${tx.txId} sender: ${tx.sender_address} nonce: ${tx.nonce}` + ); - // Also restore transactions for the same `sender_address` with the same `nonce`. - const inputData = transactions.map(t => [t.txId.replace('0x', '\\x'), t.sender, t.nonce]); + // Restore new non-canonical txs into the mempool. Also restore transactions for the same + // senders/sponsors with the same `nonce`s. We will recalculate replace-by-fee ordering shortly + // afterwards. + const inputData = transactions.map(t => [ + t.txId.replace('0x', '\\x'), + t.sender_address, + t.sponsor_address ?? 'null', + t.sponsored.toString(), + t.nonce, + ]); const updatedRows = await sql<{ tx_id: string }[]>` - WITH input_data (tx_id, sender_address, nonce) AS (VALUES ${sql(inputData)}), + WITH input_data (tx_id, sender_address, sponsor_address, sponsored, nonce) + AS (VALUES ${sql(inputData)}), affected_mempool_tx_ids AS ( SELECT m.tx_id FROM mempool_txs AS m INNER JOIN input_data AS i - ON m.sender_address = i.sender_address AND m.nonce = i.nonce::int + ON m.nonce = i.nonce::int + AND (CASE i.sponsored::boolean + WHEN true THEN (m.sponsor_address = i.sponsor_address OR m.sender_address = i.sponsor_address) + ELSE (m.sponsor_address = i.sender_address OR m.sender_address = i.sender_address) + END) UNION SELECT tx_id::bytea FROM input_data ), restored AS ( UPDATE mempool_txs - SET pruned = false, status = ${DbTxStatus.Pending} + SET pruned = false, status = ${DbTxStatus.Pending}, replaced_by_tx_id = NULL WHERE pruned = true AND tx_id IN (SELECT DISTINCT tx_id FROM affected_mempool_tx_ids) RETURNING tx_id ), @@ -2751,24 +2875,39 @@ export class PgWriteStore extends PgStore { if (transactions.length === 0) return { removedTxs: [] }; if (logger.isLevelEnabled('debug')) for (const tx of transactions) - logger.debug(`Pruning mempool tx: ${tx.txId} sender: ${tx.sender} nonce: ${tx.nonce}`); + logger.debug( + `Pruning mempool tx: ${tx.txId} sender: ${tx.sender_address} nonce: ${tx.nonce}` + ); - // Also prune transactions for the same `sender_address` with the same `nonce`. - const inputData = transactions.map(t => [t.txId.replace('0x', '\\x'), t.sender, t.nonce]); + // Prune confirmed txs from the mempool. Also prune transactions for the same senders/sponsors + // with the same `nonce`s. We'll recalculate replaced-by-fee data later when new block data is + // written to the DB. + const inputData = transactions.map(t => [ + t.txId.replace('0x', '\\x'), + t.sender_address, + t.sponsor_address ?? 'null', + t.sponsored.toString(), + t.nonce, + ]); const updateResults = await sql<{ tx_id: string }[]>` - WITH input_data (tx_id, sender_address, nonce) AS (VALUES ${sql(inputData)}), + WITH input_data (tx_id, sender_address, sponsor_address, sponsored, nonce) + AS (VALUES ${sql(inputData)}), affected_mempool_tx_ids AS ( SELECT m.tx_id FROM mempool_txs AS m INNER JOIN input_data AS i - ON m.sender_address = i.sender_address AND m.nonce = i.nonce::int + ON m.nonce = i.nonce::int + AND (CASE i.sponsored::boolean + WHEN true THEN (m.sponsor_address = i.sponsor_address OR m.sender_address = i.sponsor_address) + ELSE (m.sponsor_address = i.sender_address OR m.sender_address = i.sender_address) + END) UNION SELECT tx_id::bytea FROM input_data ), pruned AS ( UPDATE mempool_txs - SET pruned = true - WHERE pruned = false AND tx_id IN (SELECT DISTINCT tx_id FROM affected_mempool_tx_ids) + SET pruned = true, replaced_by_tx_id = NULL + WHERE pruned = false AND tx_id IN (SELECT tx_id FROM affected_mempool_tx_ids) RETURNING tx_id ), count_update AS ( @@ -2845,7 +2984,14 @@ export class PgWriteStore extends PgStore { const q = new PgWriteQueue(); q.enqueue(async () => { const txResult = await sql< - { tx_id: string; sender_address: string; nonce: number; update_balances_count: number }[] + { + tx_id: string; + sender_address: string; + sponsor_address: string | null; + sponsored: boolean; + nonce: number; + update_balances_count: number; + }[] >` WITH updated_txs AS ( UPDATE txs @@ -2886,12 +3032,15 @@ export class PgWriteStore extends PgStore { SET balance = ft_balances.balance + EXCLUDED.balance RETURNING ft_balances.address ) - SELECT tx_id, sender_address, nonce, (SELECT COUNT(*)::int FROM update_ft_balances) AS update_balances_count + SELECT tx_id, sender_address, sponsor_address, sponsored, nonce, + (SELECT COUNT(*)::int FROM update_ft_balances) AS update_balances_count FROM updated_txs `; const txs = txResult.map(row => ({ txId: row.tx_id, - sender: row.sender_address, + sender_address: row.sender_address, + sponsor_address: row.sponsor_address ?? undefined, + sponsored: row.sponsored, nonce: row.nonce, })); if (canonical) { diff --git a/src/event-stream/core-node-message.ts b/src/event-stream/core-node-message.ts index 7167aa7d4..9a1cc3fd5 100644 --- a/src/event-stream/core-node-message.ts +++ b/src/event-stream/core-node-message.ts @@ -381,6 +381,7 @@ export type CoreNodeDropMempoolTxReasonType = export interface CoreNodeDropMempoolTxMessage { dropped_txids: string[]; reason: CoreNodeDropMempoolTxReasonType; + new_txid: string | null; } export interface CoreNodeAttachmentMessage { diff --git a/src/event-stream/event-server.ts b/src/event-stream/event-server.ts index 11a3fbd6a..97a9e6d2f 100644 --- a/src/event-stream/event-server.ts +++ b/src/event-stream/event-server.ts @@ -170,7 +170,11 @@ async function handleDroppedMempoolTxsMessage( ): Promise { logger.debug(`Received ${msg.dropped_txids.length} dropped mempool txs`); const dbTxStatus = getTxDbStatus(msg.reason); - await db.dropMempoolTxs({ status: dbTxStatus, txIds: msg.dropped_txids }); + await db.dropMempoolTxs({ + status: dbTxStatus, + txIds: msg.dropped_txids, + new_tx_id: msg.new_txid, + }); } async function handleMicroblockMessage( diff --git a/tests/api/address.test.ts b/tests/api/address.test.ts index 1f5547c94..9df6b680f 100644 --- a/tests/api/address.test.ts +++ b/tests/api/address.test.ts @@ -2645,6 +2645,7 @@ describe('address tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: true, @@ -2695,10 +2696,11 @@ describe('address tests', () => { const mempoolTx1: DbMempoolTxRaw = { tx_id: '0x52123456', anchor_mode: 3, - nonce: 1, + nonce: 6, raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: true, diff --git a/tests/api/cache-control.test.ts b/tests/api/cache-control.test.ts index cab1c8d9c..2fbc3de3c 100644 --- a/tests/api/cache-control.test.ts +++ b/tests/api/cache-control.test.ts @@ -380,7 +380,11 @@ describe('cache-control tests', () => { expect(request3.text).toBe(''); // Drop one tx. - await db.dropMempoolTxs({ status: DbTxStatus.DroppedReplaceByFee, txIds: ['0x1101'] }); + await db.dropMempoolTxs({ + status: DbTxStatus.DroppedReplaceByFee, + txIds: ['0x1101'], + new_tx_id: '0x1109', + }); // Cache is now a miss. const request4 = await supertest(api.server) @@ -753,7 +757,9 @@ describe('cache-control tests', () => { // Add STX tx. await db.updateMempoolTxs({ - mempoolTxs: [testMempoolTx({ tx_id: '0x0001', receipt_time: 1000, sender_address })], + mempoolTxs: [ + testMempoolTx({ tx_id: '0x0001', receipt_time: 1000, sender_address, nonce: 0 }), + ], }); // Valid ETag. @@ -772,7 +778,12 @@ describe('cache-control tests', () => { // Add sponsor tx. await db.updateMempoolTxs({ mempoolTxs: [ - testMempoolTx({ tx_id: '0x0002', receipt_time: 2000, sponsor_address: sender_address }), + testMempoolTx({ + tx_id: '0x0002', + receipt_time: 2000, + sponsor_address: sender_address, + nonce: 1, + }), ], }); @@ -795,6 +806,7 @@ describe('cache-control tests', () => { tx_id: '0x0003', receipt_time: 3000, token_transfer_recipient_address: sender_address, + nonce: 2, }), ], }); @@ -813,7 +825,7 @@ describe('cache-control tests', () => { // Change mempool with no changes to this address. await db.updateMempoolTxs({ - mempoolTxs: [testMempoolTx({ tx_id: '0x0004', receipt_time: 4000 })], + mempoolTxs: [testMempoolTx({ tx_id: '0x0004', receipt_time: 4000, nonce: 3 })], }); // Cache still works. diff --git a/tests/api/datastore.test.ts b/tests/api/datastore.test.ts index 7f5a39969..f91eeffc7 100644 --- a/tests/api/datastore.test.ts +++ b/tests/api/datastore.test.ts @@ -3363,6 +3363,7 @@ describe('postgres datastore', () => { token_transfer_memo: bufferToHex(Buffer.from('hi')), token_transfer_recipient_address: 'stx-recipient-addr', status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x', fee_rate: 1234n, sponsored: false, diff --git a/tests/api/mempool.test.ts b/tests/api/mempool.test.ts index cd8f128cc..3a06e3c2b 100644 --- a/tests/api/mempool.test.ts +++ b/tests/api/mempool.test.ts @@ -57,7 +57,11 @@ describe('mempool tests', () => { .addTx({ tx_id: `0x11${hexFromHeight(block_height)}`, nonce: block_height }) .build(); await db.update(block); - const mempoolTx = testMempoolTx({ tx_id: `0x${hexFromHeight(block_height)}` }); + const mempoolTx = testMempoolTx({ + tx_id: `0x${hexFromHeight(block_height)}`, + nonce: block_height, + sender_address: 'SP3SBQ9PZEMBNBAWTR7FRPE3XK0EFW9JWVX4G80S2', + }); await db.updateMempoolTxs({ mempoolTxs: [mempoolTx] }); } await db.update( @@ -120,7 +124,7 @@ describe('mempool tests', () => { index_block_hash: `0x0${block_height}`, parent_index_block_hash: `0x0${block_height - 1}`, }) - .addTx({ tx_id: `0x111${block_height}` }) + .addTx({ tx_id: `0x111${block_height}`, nonce: block_height }) .build(); await db.update(block); const mempoolTx1 = testMempoolTx({ @@ -129,20 +133,23 @@ describe('mempool tests', () => { fee_rate: BigInt(100 * block_height), raw_tx: '0x' + 'ff'.repeat(block_height), nonce: block_height, + sender_address: 'SP3SBQ9PZEMBNBAWTR7FRPE3XK0EFW9JWVX4G80S2', }); const mempoolTx2 = testMempoolTx({ tx_id: `0x1${block_height}`, type_id: DbTxTypeId.ContractCall, fee_rate: BigInt(200 * block_height), raw_tx: '0x' + 'ff'.repeat(block_height + 10), - nonce: block_height + 1, + nonce: block_height, + sender_address: 'SP3XXK8BG5X7CRH7W07RRJK3JZJXJ799WX3Y0SMCR', }); const mempoolTx3 = testMempoolTx({ tx_id: `0x2${block_height}`, type_id: DbTxTypeId.SmartContract, fee_rate: BigInt(300 * block_height), raw_tx: '0x' + 'ff'.repeat(block_height + 20), - nonce: block_height + 2, + nonce: block_height, + sender_address: 'SPM0SBD3R79CDZ3AWBD3BRQS13JZA47PK0207K94', }); await db.updateMempoolTxs({ mempoolTxs: [mempoolTx1, mempoolTx2, mempoolTx3] }); } @@ -190,6 +197,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, receipt_time: 1594307695, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), post_conditions: '0x01f5', @@ -207,6 +215,7 @@ describe('mempool tests', () => { const expectedResp1 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', fee_rate: '1234', nonce: 0, @@ -234,6 +243,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.VersionedSmartContract, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, receipt_time: 1594307695, smart_contract_clarity_version: 2, smart_contract_contract_id: 'some-versioned-smart-contract', @@ -254,6 +264,7 @@ describe('mempool tests', () => { const expectedResp1 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'smart_contract', fee_rate: '1234', nonce: 0, @@ -285,6 +296,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, receipt_time: 1594307695, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), post_conditions: '0x01f5', @@ -302,6 +314,7 @@ describe('mempool tests', () => { const expectedResp1 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', fee_rate: '1234', nonce: 0, @@ -330,6 +343,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, receipt_time: 1594307695, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), post_conditions: '0x01f5', @@ -343,34 +357,44 @@ describe('mempool tests', () => { ...mempoolTx1, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000001', receipt_time: 1594307702, + nonce: 1, }; const mempoolTx3: DbMempoolTxRaw = { ...mempoolTx1, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000003', receipt_time: 1594307703, + nonce: 2, }; const mempoolTx4: DbMempoolTxRaw = { ...mempoolTx1, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000004', receipt_time: 1594307704, + nonce: 3, }; const mempoolTx5: DbMempoolTxRaw = { ...mempoolTx1, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', receipt_time: 1594307705, + nonce: 4, }; const mempoolTx6: DbMempoolTxRaw = { ...mempoolTx1, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', receipt_time: 1594307706, + nonce: 5, }; + const new_txid1: string = '0x8912000000000000000000000000000000000000000000000000000000000099'; + + const new_txid2: string = '0x8912000000000000000000000000000000000000000000000000000000000100'; + await db.updateMempoolTxs({ mempoolTxs: [mempoolTx1, mempoolTx2, mempoolTx3, mempoolTx4, mempoolTx5, mempoolTx6], }); await db.dropMempoolTxs({ status: DbTxStatus.DroppedReplaceAcrossFork, txIds: [mempoolTx1.tx_id, mempoolTx2.tx_id], + new_tx_id: new_txid1, }); const searchResult1 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx1.tx_id}`); @@ -379,6 +403,7 @@ describe('mempool tests', () => { const expectedResp1 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'dropped_replace_across_fork', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000099', tx_type: 'coinbase', fee_rate: '1234', nonce: 0, @@ -400,9 +425,10 @@ describe('mempool tests', () => { const expectedResp2 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000001', tx_status: 'dropped_replace_across_fork', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000099', tx_type: 'coinbase', fee_rate: '1234', - nonce: 0, + nonce: 1, anchor_mode: 'any', sender_address: 'sender-addr', sponsor_address: 'sponsor-addr', @@ -419,6 +445,7 @@ describe('mempool tests', () => { await db.dropMempoolTxs({ status: DbTxStatus.DroppedReplaceByFee, txIds: [mempoolTx3.tx_id], + new_tx_id: new_txid2, }); const searchResult3 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx3.tx_id}`); expect(searchResult3.status).toBe(200); @@ -426,9 +453,10 @@ describe('mempool tests', () => { const expectedResp3 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000003', tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000100', tx_type: 'coinbase', fee_rate: '1234', - nonce: 0, + nonce: 2, anchor_mode: 'any', sender_address: 'sender-addr', sponsor_address: 'sponsor-addr', @@ -444,6 +472,7 @@ describe('mempool tests', () => { await db.dropMempoolTxs({ status: DbTxStatus.DroppedTooExpensive, txIds: [mempoolTx4.tx_id], + new_tx_id: null, }); const searchResult4 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx4.tx_id}`); expect(searchResult4.status).toBe(200); @@ -451,9 +480,10 @@ describe('mempool tests', () => { const expectedResp4 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000004', tx_status: 'dropped_too_expensive', + replaced_by_tx_id: null, tx_type: 'coinbase', fee_rate: '1234', - nonce: 0, + nonce: 3, anchor_mode: 'any', sender_address: 'sender-addr', sponsor_address: 'sponsor-addr', @@ -469,6 +499,7 @@ describe('mempool tests', () => { await db.dropMempoolTxs({ status: DbTxStatus.DroppedStaleGarbageCollect, txIds: [mempoolTx5.tx_id], + new_tx_id: null, }); const searchResult5 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx5.tx_id}`); expect(searchResult5.status).toBe(200); @@ -476,9 +507,10 @@ describe('mempool tests', () => { const expectedResp5 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'dropped_stale_garbage_collect', + replaced_by_tx_id: null, tx_type: 'coinbase', fee_rate: '1234', - nonce: 0, + nonce: 4, anchor_mode: 'any', sender_address: 'sender-addr', sponsor_address: 'sponsor-addr', @@ -494,6 +526,7 @@ describe('mempool tests', () => { await db.dropMempoolTxs({ status: DbTxStatus.DroppedProblematic, txIds: [mempoolTx6.tx_id], + new_tx_id: null, }); const searchResult6 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx6.tx_id}`); expect(searchResult6.status).toBe(200); @@ -501,9 +534,10 @@ describe('mempool tests', () => { const expectedResp6 = { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'dropped_problematic', + replaced_by_tx_id: null, tx_type: 'coinbase', fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', sender_address: 'sender-addr', sponsor_address: 'sponsor-addr', @@ -527,26 +561,32 @@ describe('mempool tests', () => { expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'dropped_problematic', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'dropped_stale_garbage_collect', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000004', tx_status: 'dropped_too_expensive', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000003', tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000100', }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000001', tx_status: 'dropped_replace_across_fork', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000099', }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'dropped_replace_across_fork', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000099', }), ]), }) @@ -632,22 +672,27 @@ describe('mempool tests', () => { expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'dropped_problematic', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'dropped_stale_garbage_collect', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000004', tx_status: 'dropped_too_expensive', + replaced_by_tx_id: null, }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000003', tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000100', }), expect.objectContaining({ tx_id: '0x8912000000000000000000000000000000000000000000000000000000000001', tx_status: 'dropped_replace_across_fork', + replaced_by_tx_id: '0x8912000000000000000000000000000000000000000000000000000000000099', }), ]), }) @@ -662,12 +707,13 @@ describe('mempool tests', () => { pruned: false, tx_id: `0x891200000000000000000000000000000000000000000000000000000000000${i}`, anchor_mode: 3, - nonce: 0, + nonce: i, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, receipt_time: (new Date(`2020-07-09T15:14:0${i}Z`).getTime() / 1000) | 0, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -690,11 +736,12 @@ describe('mempool tests', () => { { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000007', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', receipt_time: 1594307647, receipt_time_iso: '2020-07-09T15:14:07.000Z', fee_rate: '1234', - nonce: 0, + nonce: 7, anchor_mode: 'any', sender_address: 'sender-addr', sponsored: false, @@ -705,11 +752,12 @@ describe('mempool tests', () => { { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', receipt_time: 1594307646, receipt_time_iso: '2020-07-09T15:14:06.000Z', fee_rate: '1234', - nonce: 0, + nonce: 6, anchor_mode: 'any', sender_address: 'sender-addr', sponsored: false, @@ -720,11 +768,12 @@ describe('mempool tests', () => { { tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', receipt_time: 1594307645, receipt_time_iso: '2020-07-09T15:14:05.000Z', fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', sender_address: 'sender-addr', sponsored: false, @@ -801,11 +850,12 @@ describe('mempool tests', () => { pruned: false, tx_id: `0x89120000000000000000000000000000000000000000000000000000000000${paddedIndex}`, anchor_mode: 3, - nonce: 0, + nonce: index, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: xfer.type_id, receipt_time: (new Date(`2020-07-09T15:14:${paddedIndex}Z`).getTime() / 1000) | 0, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -835,7 +885,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 6, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -850,11 +900,12 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, { fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -869,6 +920,7 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, ], @@ -887,7 +939,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 7, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -902,11 +954,12 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000007', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, { fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -921,6 +974,7 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, ], @@ -939,7 +993,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -954,6 +1008,7 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, ], @@ -972,7 +1027,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 6, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -987,11 +1042,12 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000006', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, { fee_rate: '1234', - nonce: 0, + nonce: 5, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1006,6 +1062,7 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000005', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, ], @@ -1024,7 +1081,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 10, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1039,11 +1096,12 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000010', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, { fee_rate: '1234', - nonce: 0, + nonce: 8, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1053,6 +1111,7 @@ describe('mempool tests', () => { sponsored: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000008', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'contract_call', contract_call: { contract_id: 'SP32AEEF6WW5Y0NMJ1S8SBSZDAY8R5J32NBZFPKKZ.free-punks-v0', @@ -1076,7 +1135,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 10, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1091,11 +1150,12 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000010', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, { fee_rate: '1234', - nonce: 0, + nonce: 8, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1105,6 +1165,7 @@ describe('mempool tests', () => { sponsored: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000008', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'contract_call', contract_call: { contract_id: 'SP32AEEF6WW5Y0NMJ1S8SBSZDAY8R5J32NBZFPKKZ.free-punks-v0', @@ -1128,7 +1189,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 9, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1138,6 +1199,7 @@ describe('mempool tests', () => { sponsored: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000009', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'smart_contract', smart_contract: { clarity_version: null, @@ -1161,7 +1223,7 @@ describe('mempool tests', () => { results: [ { fee_rate: '1234', - nonce: 0, + nonce: 10, anchor_mode: 'any', post_condition_mode: 'allow', post_conditions: [], @@ -1176,6 +1238,7 @@ describe('mempool tests', () => { }, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000010', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'token_transfer', }, ], @@ -1196,11 +1259,12 @@ describe('mempool tests', () => { pruned: false, tx_id: `0x89120000000000000000000000000000000000000000000000000000000000${paddedIndex}`, anchor_mode: 3, - nonce: 0, + nonce: index, raw_tx: bufferToHex(Buffer.from('x'.repeat(index + 1))), type_id: DbTxTypeId.TokenTransfer, receipt_time: (new Date(`2020-07-09T15:14:${paddedIndex}Z`).getTime() / 1000) | 0, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 100n * BigInt(index + 1), sponsored: false, @@ -1268,7 +1332,7 @@ describe('mempool tests', () => { test('mempool - contract_call tx abi details are retrieved', async () => { const block1 = new TestBlockBuilder() - .addTx() + .addTx({ nonce: 0 }) .addTxSmartContract() .addTxContractLogEvent() .build(); @@ -1277,6 +1341,7 @@ describe('mempool tests', () => { const mempoolTx1 = testMempoolTx({ type_id: DbTxTypeId.ContractCall, tx_id: '0x1232000000000000000000000000000000000000000000000000000000000000', + nonce: 1, }); await db.updateMempoolTxs({ mempoolTxs: [mempoolTx1] }); @@ -1322,10 +1387,13 @@ describe('mempool tests', () => { expectedContractDetails ); + const new_txid: string = '0x1232000000000000000000000000000000000000000000000000000000000001'; + // Dropped mempool tx await db.dropMempoolTxs({ status: DbTxStatus.DroppedReplaceAcrossFork, txIds: [mempoolTx1.tx_id], + new_tx_id: new_txid, }); const mempoolDropResults = await supertest(api.server).get(`/extended/v1/tx/mempool/dropped`); expect(mempoolDropResults.status).toBe(200); @@ -1370,6 +1438,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1397,6 +1466,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1448,6 +1518,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1470,6 +1541,7 @@ describe('mempool tests', () => { { tx_id: '0x521234', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', receipt_time: 1616063078, receipt_time_iso: '2021-03-18T10:24:38.000Z', @@ -1701,6 +1773,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1880,6 +1953,7 @@ describe('mempool tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1928,6 +2002,7 @@ describe('mempool tests', () => { await db.dropMempoolTxs({ status: DbTxStatus.DroppedStaleGarbageCollect, txIds: [mempoolTx.tx_id], + new_tx_id: '', }); // Verify tx is pruned from mempool @@ -2047,6 +2122,7 @@ describe('mempool tests', () => { anchor_mode: 3, raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', sponsored: false, sponsor_address: undefined, @@ -2067,6 +2143,7 @@ describe('mempool tests', () => { anchor_mode: 3, raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', sponsored: false, sponsor_address: undefined, @@ -2086,6 +2163,7 @@ describe('mempool tests', () => { anchor_mode: 3, raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', sponsored: false, sponsor_address: undefined, @@ -2129,8 +2207,8 @@ describe('mempool tests', () => { }); }); - test('prunes transactions with nonces that were already confirmed', async () => { - // Initial block + test('prunes and restores replaced-by-fee transactions', async () => { + const sender_address = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; await db.update( new TestBlockBuilder({ block_height: 1, @@ -2140,31 +2218,108 @@ describe('mempool tests', () => { ); // Add tx with nonce = 1 to the mempool - const sender_address = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; - const mempoolTx = testMempoolTx({ tx_id: `0xff0001`, sender_address, nonce: 1 }); - await db.updateMempoolTxs({ mempoolTxs: [mempoolTx] }); - const check0 = await supertest(api.server).get( - `/extended/v1/address/${sender_address}/mempool` + await db.updateMempoolTxs({ + mempoolTxs: [ + testMempoolTx({ + tx_id: `0xff0001`, + sender_address, + nonce: 1, + fee_rate: 200n, + type_id: DbTxTypeId.TokenTransfer, + }), + ], + }); + let request = await supertest(api.server).get(`/extended/v1/tx/mempool`); + expect(request.body.total).toBe(1); + expect(request.body.results).toHaveLength(1); + + // Add another tx with nonce = 1 to the mempool with a higher fee. Previous tx is marked as + // pruned and replaced. + await db.updateMempoolTxs({ + mempoolTxs: [ + testMempoolTx({ + tx_id: `0xff0002`, + sender_address, + nonce: 1, + fee_rate: 300n, + type_id: DbTxTypeId.TokenTransfer, + }), + ], + }); + request = await supertest(api.server).get(`/extended/v1/tx/mempool`); + expect(request.body.total).toBe(1); + expect(request.body.results).toHaveLength(1); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0001`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xff0002', + }) ); - expect(check0.body.results).toHaveLength(1); - // Confirm a different tx with the same nonce = 1 by the same sender without it ever touching - // the mempool + // Add yet another conflicting tx but our address is the sponsor. Since it has a lower fee, it + // will be immediately marked as RBFd by 0xff0002. + await db.updateMempoolTxs({ + mempoolTxs: [ + testMempoolTx({ + tx_id: `0xff0003`, + sender_address: 'SP3FXEKSA6D4BW3TFP2BWTSREV6FY863Y90YY7D8G', + sponsor_address: sender_address, + sponsored: true, + nonce: 1, + fee_rate: 150n, + type_id: DbTxTypeId.TokenTransfer, + }), + ], + }); + request = await supertest(api.server).get(`/extended/v1/tx/mempool`); + expect(request.body.total).toBe(1); + expect(request.body.results).toHaveLength(1); + expect(request.body.results[0].tx_id).toBe('0xff0002'); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0003`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xff0002', + }) + ); + + // Confirm a block containing a new tx with the same nonce = 1 by the same sender without it + // ever touching the mempool await db.update( new TestBlockBuilder({ block_height: 2, index_block_hash: `0x0002`, parent_index_block_hash: `0x0001`, }) - .addTx({ tx_id: `0xaa0001`, sender_address, nonce: 1 }) + .addTx({ + tx_id: `0xaa0001`, + sender_address, + nonce: 1, + fee_rate: 100n, + type_id: DbTxTypeId.TokenTransfer, + }) .build() ); - // Mempool tx should now be pruned - const check1 = await supertest(api.server).get( - `/extended/v1/address/${sender_address}/mempool` + // Old mempool txs are now pruned and both marked as replaced by the confirmed tx. + request = await supertest(api.server).get(`/extended/v1/tx/mempool`); + expect(request.body.total).toBe(0); + expect(request.body.results).toHaveLength(0); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0001`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xaa0001', + }) + ); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0002`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xaa0001', + }) ); - expect(check1.body.results).toHaveLength(0); // Re-org block 2 await db.update( @@ -2182,11 +2337,39 @@ describe('mempool tests', () => { }).build() ); - // Now both conflicting nonce txs should coexist in the mempool (like RBF) - const check2 = await supertest(api.server).get( - `/extended/v1/address/${sender_address}/mempool` + // Only the highest fee tx is restored to the mempool, and all others are pruned and marked as + // RBFd by it. + request = await supertest(api.server).get(`/extended/v1/tx/mempool`); + expect(request.body.total).toBe(1); + expect(request.body.results).toHaveLength(1); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0002`); // Winner + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'pending', + replaced_by_tx_id: null, + }) + ); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0001`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xff0002', + }) + ); + request = await supertest(api.server).get(`/extended/v1/tx/0xaa0001`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xff0002', + }) + ); + request = await supertest(api.server).get(`/extended/v1/tx/0xff0003`); + expect(request.body).toEqual( + expect.objectContaining({ + tx_status: 'dropped_replace_by_fee', + replaced_by_tx_id: '0xff0002', + }) ); - expect(check2.body.results).toHaveLength(2); }); test('account estimated balance from mempool activity', async () => { @@ -2222,6 +2405,7 @@ describe('mempool tests', () => { sender_address: address, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 0, }), ], }); @@ -2243,6 +2427,7 @@ describe('mempool tests', () => { contract_call_function_args: '', contract_call_function_name: 'test', fee_rate: 50n, + nonce: 1, }), ], }); @@ -2261,6 +2446,7 @@ describe('mempool tests', () => { sponsored: true, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 2, }), ], }); @@ -2278,6 +2464,7 @@ describe('mempool tests', () => { token_transfer_recipient_address: address, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 1, }), ], }); @@ -2307,6 +2494,7 @@ describe('mempool tests', () => { sender_address: address, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 0, }) .addTxStxEvent({ sender: address, amount: 100n }) .addTx({ @@ -2318,6 +2506,7 @@ describe('mempool tests', () => { contract_call_function_args: '', contract_call_function_name: 'test', fee_rate: 50n, + nonce: 1, }) .addTx({ tx_id: '0x0003', @@ -2325,12 +2514,14 @@ describe('mempool tests', () => { sponsored: true, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 2, }) .addTx({ tx_id: '0x0004', token_transfer_recipient_address: address, token_transfer_amount: 100n, fee_rate: 50n, + nonce: 1, }) .addTxStxEvent({ recipient: address, amount: 100n }) .build() diff --git a/tests/api/microblock.test.ts b/tests/api/microblock.test.ts index 902af7e96..685b266b8 100644 --- a/tests/api/microblock.test.ts +++ b/tests/api/microblock.test.ts @@ -417,7 +417,7 @@ describe('microblock tests', () => { tx_id: '0x02', tx_index: 0, anchor_mode: 3, - nonce: 0, + nonce: 1, raw_tx: '0x141414', type_id: DbTxTypeId.TokenTransfer, status: 1, @@ -460,7 +460,7 @@ describe('microblock tests', () => { tx_id: '0x03', tx_index: 1, anchor_mode: 3, - nonce: 0, + nonce: 2, raw_tx: '0x141415', type_id: DbTxTypeId.ContractCall, status: 1, @@ -509,11 +509,13 @@ describe('microblock tests', () => { const mempoolTx1: DbMempoolTxRaw = { ...mbTx1, pruned: false, + replaced_by_tx_id: undefined, receipt_time: 123456789, }; const mempoolTx2: DbMempoolTxRaw = { ...mbTx2, pruned: false, + replaced_by_tx_id: undefined, receipt_time: 123456789, }; await db.updateMempoolTxs({ mempoolTxs: [mempoolTx1, mempoolTx2] }); diff --git a/tests/api/search.test.ts b/tests/api/search.test.ts index 732feadb1..ab4ad1607 100644 --- a/tests/api/search.test.ts +++ b/tests/api/search.test.ts @@ -112,12 +112,13 @@ describe('search tests', () => { pruned: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', anchor_mode: 3, - nonce: 0, + nonce: 1, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, receipt_time: 123456, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -327,12 +328,13 @@ describe('search tests', () => { pruned: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', anchor_mode: 3, - nonce: 0, + nonce: 1, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, receipt_time: 123456, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -478,7 +480,7 @@ describe('search tests', () => { alt_recipient: null, }, fee_rate: '1234', - nonce: 0, + nonce: 1, post_condition_mode: 'allow', post_conditions: [], receipt_time: 123456, @@ -487,6 +489,7 @@ describe('search tests', () => { sponsored: false, tx_id: '0x8912000000000000000000000000000000000000000000000000000000000000', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'coinbase', }, }, @@ -972,12 +975,13 @@ describe('search tests', () => { type_id: DbTxTypeId.SmartContract, tx_id: '0x1111882200000000000000000000000000000000000000000000000000000000', anchor_mode: 3, - nonce: 0, + nonce: 1, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), receipt_time: 123456, smart_contract_contract_id: contractAddr2, smart_contract_source_code: '(some-src)', status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1600,12 +1604,13 @@ describe('search tests', () => { type_id: DbTxTypeId.SmartContract, tx_id: '0x1111882200000000000000000000000000000000000000000000000000000000', anchor_mode: 3, - nonce: 0, + nonce: 1, raw_tx: bufferToHex(Buffer.from('test-raw-tx')), receipt_time: 123456, smart_contract_contract_id: contractAddr2, smart_contract_source_code: '(some-src)', status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -1633,7 +1638,7 @@ describe('search tests', () => { metadata: { anchor_mode: 'any', fee_rate: '1234', - nonce: 0, + nonce: 1, post_condition_mode: 'allow', post_conditions: [], receipt_time: 123456, @@ -1647,6 +1652,7 @@ describe('search tests', () => { sponsored: false, tx_id: '0x1111882200000000000000000000000000000000000000000000000000000000', tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'smart_contract', }, }, diff --git a/tests/api/tx.test.ts b/tests/api/tx.test.ts index 99803a3f5..9b4b50737 100644 --- a/tests/api/tx.test.ts +++ b/tests/api/tx.test.ts @@ -78,6 +78,7 @@ describe('tx tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-tx')), type_id: DbTxTypeId.Coinbase, status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, receipt_time: 1594307695, coinbase_payload: bufferToHex(Buffer.from('coinbase hi')), post_conditions: '0x01f5', @@ -2780,6 +2781,7 @@ describe('tx tests', () => { raw_tx: bufferToHex(Buffer.from('test-raw-mempool-tx')), type_id: DbTxTypeId.Coinbase, status: 1, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 1234n, sponsored: false, @@ -3681,7 +3683,7 @@ describe('tx tests', () => { type_id: DbTxTypeId.ContractCall, tx_id: '0x1513739d6a3f86d4597f5296cc536f6890e2affff9aece285e37399be697b43f', anchor_mode: DbTxAnchorMode.Any, - nonce: 0, + nonce: 1, raw_tx: bufferToHex(Buffer.from('')), canonical: true, microblock_canonical: true, @@ -3865,7 +3867,7 @@ describe('tx tests', () => { const expected2 = { tx_id: '0x1513739d6a3f86d4597f5296cc536f6890e2affff9aece285e37399be697b43f', - nonce: 0, + nonce: 1, fee_rate: '139200', sender_address: 'SPX3DV9X9CGA8P14B3CMP2X8DBW6ZDXEAXDNPTER', sponsored: false, @@ -3960,9 +3962,10 @@ describe('tx tests', () => { type_id: DbTxTypeId.ContractCall, tx_id: '0x4413739d6a3f86d4597f5296cc536f6890e2affff9aece285e37399be697b43f', anchor_mode: DbTxAnchorMode.Any, - nonce: 0, + nonce: 2, raw_tx: bufferToHex(Buffer.from('')), status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 139200n, sponsored: false, @@ -4001,7 +4004,7 @@ describe('tx tests', () => { ], }, fee_rate: '139200', - nonce: 0, + nonce: 2, post_condition_mode: 'allow', post_conditions: [], receipt_time: 0, @@ -4010,6 +4013,7 @@ describe('tx tests', () => { sponsored: false, tx_id: mempoolTx1.tx_id, tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'contract_call', }; const mempoolTxResult1 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx1.tx_id}`); @@ -4020,9 +4024,10 @@ describe('tx tests', () => { type_id: DbTxTypeId.ContractCall, tx_id: '0x5513739d6a3f86d4597f5296cc536f6890e2affff9aece285e37399be697b43f', anchor_mode: DbTxAnchorMode.Any, - nonce: 0, + nonce: 3, raw_tx: bufferToHex(Buffer.from('')), status: DbTxStatus.Pending, + replaced_by_tx_id: undefined, post_conditions: '0x01f5', fee_rate: 139200n, sponsored: false, @@ -4062,7 +4067,7 @@ describe('tx tests', () => { '(define-public (bns-name-preorder (hashedSaltedFqn (buff 20)) (stxToBurn uint) (paymentSIP010Trait trait_reference) (reciprocityTokenTrait trait_reference) (referencerWallet principal)))', }, fee_rate: '139200', - nonce: 0, + nonce: 3, post_condition_mode: 'allow', post_conditions: [], receipt_time: 0, @@ -4071,6 +4076,7 @@ describe('tx tests', () => { sponsored: false, tx_id: mempoolTx2.tx_id, tx_status: 'pending', + replaced_by_tx_id: null, tx_type: 'contract_call', }; const mempoolTxResult2 = await supertest(api.server).get(`/extended/v1/tx/${mempoolTx2.tx_id}`); diff --git a/tests/rosetta/api.test.ts b/tests/rosetta/api.test.ts index 8ba073993..505acb8ed 100644 --- a/tests/rosetta/api.test.ts +++ b/tests/rosetta/api.test.ts @@ -969,7 +969,7 @@ describe('Rosetta API', () => { pruned: false, tx_id: `0x891200000000000000000000000000000000000000000000000000000000000${i}`, anchor_mode: 3, - nonce: 0, + nonce: i, raw_tx: '0x6655443322', type_id: DbTxTypeId.Coinbase, receipt_time: (new Date(`2020-07-09T15:14:0${i}Z`).getTime() / 1000) | 0, diff --git a/tests/utils/test-builders.ts b/tests/utils/test-builders.ts index 03c8b4c53..d1f51792c 100644 --- a/tests/utils/test-builders.ts +++ b/tests/utils/test-builders.ts @@ -307,6 +307,7 @@ interface TestMempoolTxArgs { smart_contract_clarity_version?: number; smart_contract_contract_id?: string; status?: DbTxStatus; + replaced_by_tx_id?: string; token_transfer_recipient_address?: string; token_transfer_amount?: bigint; token_transfer_memo?: string; @@ -335,6 +336,7 @@ export function testMempoolTx(args?: TestMempoolTxArgs): DbMempoolTxRaw { type_id: args?.type_id ?? DbTxTypeId.TokenTransfer, receipt_time: args?.receipt_time ?? (new Date().getTime() / 1000) | 0, status: args?.status ?? DbTxStatus.Pending, + replaced_by_tx_id: args?.replaced_by_tx_id, post_conditions: '0x01f5', fee_rate: args?.fee_rate ?? 1234n, sponsored: args?.sponsored ?? false,