diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 50241a31ff..7d9ff093db 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1006,6 +1006,20 @@ paths: schema: type: string example: "SP6P4EJF0VG8V0RB3TQQKJBHDQKEF6NVRD1KZE3C.satoshibles" + - name: contains + in: query + description: Optional stringified JSON to select only results that contain the given JSON + required: false + schema: + type: string + example: '{"attachment":{"metadata":{"op":"name-register"}}}' + - name: filter_path + in: query + description: Optional [`jsonpath` expression](https://www.postgresql.org/docs/14/functions-json.html#FUNCTIONS-SQLJSON-PATH) to select only results that contain items matching the expression + required: false + schema: + type: string + example: '$.attachment.metadata?(@.op=="name-register")' - name: limit in: query description: max number of contract events to fetch diff --git a/migrations/1680181889941_contract_log_json.js b/migrations/1680181889941_contract_log_json.js new file mode 100644 index 0000000000..2fbf562b49 --- /dev/null +++ b/migrations/1680181889941_contract_log_json.js @@ -0,0 +1,14 @@ +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.up = async pgm => { + pgm.addColumn('contract_logs', { + value_json: { + type: 'jsonb', + }, + }); +} + +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.down = pgm => { + pgm.dropIndex('contract_logs', 'value_json_path_ops_idx'); + pgm.dropColumn('contract_logs', 'value_json'); +} diff --git a/package-lock.json b/package-lock.json index d56b073d3d..243216bf75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpath-pg": "1.0.1", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", @@ -8072,6 +8073,11 @@ "node >= 0.2.0" ] }, + "node_modules/jsonpath-pg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsonpath-pg/-/jsonpath-pg-1.0.1.tgz", + "integrity": "sha512-fx0cqOxczvh2HdyBGSRrTfzfRTmvS4gMFlOb13Q96c0TOv7f9uzEmdUE1TWhBJhanbOtxA3Oy11LNd87L6Nnrg==" + }, "node_modules/jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", @@ -18241,6 +18247,11 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "dev": true }, + "jsonpath-pg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsonpath-pg/-/jsonpath-pg-1.0.1.tgz", + "integrity": "sha512-fx0cqOxczvh2HdyBGSRrTfzfRTmvS4gMFlOb13Q96c0TOv7f9uzEmdUE1TWhBJhanbOtxA3Oy11LNd87L6Nnrg==" + }, "jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", diff --git a/package.json b/package.json index 8a1720d8e0..890a4b83c2 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpath-pg": "1.0.1", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", diff --git a/src/api/query-helpers.ts b/src/api/query-helpers.ts index 56f938f3dd..c42c50f7c7 100644 --- a/src/api/query-helpers.ts +++ b/src/api/query-helpers.ts @@ -3,6 +3,7 @@ import { NextFunction, Request, Response } from 'express'; import { has0xPrefix, hexToBuffer, parseEventTypeStrings, isValidPrincipal } from './../helpers'; import { InvalidRequestError, InvalidRequestErrorType } from '../errors'; import { DbEventTypeId } from './../datastore/common'; +import { jsonpathToAst, JsonpathAst, JsonpathItem } from 'jsonpath-pg'; function handleBadRequest(res: Response, next: NextFunction, errorMessage: string): never { const error = new InvalidRequestError(errorMessage, InvalidRequestErrorType.bad_request); @@ -11,6 +12,159 @@ function handleBadRequest(res: Response, next: NextFunction, errorMessage: strin throw error; } +export function validateJsonPathQuery( + req: Request, + res: Response, + next: NextFunction, + paramName: string, + args: { paramRequired: TRequired; maxCharLength: number } +): TRequired extends true ? string | never : string | null { + if (!(paramName in req.query)) { + if (args.paramRequired) { + handleBadRequest(res, next, `Request is missing required "${paramName}" query parameter`); + } else { + return null as TRequired extends true ? string | never : string | null; + } + } + const jsonPathInput = req.query[paramName]; + if (typeof jsonPathInput !== 'string') { + handleBadRequest( + res, + next, + `Unexpected type for '${paramName}' parameter: ${JSON.stringify(jsonPathInput)}` + ); + } + + const maxCharLength = args.maxCharLength; + + if (jsonPathInput.length > maxCharLength) { + handleBadRequest( + res, + next, + `JsonPath parameter '${paramName}' is invalid: char length exceeded, max=${maxCharLength}, received=${jsonPathInput.length}` + ); + } + + let ast: JsonpathAst; + try { + ast = jsonpathToAst(jsonPathInput); + } catch (error) { + handleBadRequest(res, next, `JsonPath parameter '${paramName}' is invalid: ${error}`); + } + const astComplexity = calculateJsonpathComplexity(ast); + if (typeof astComplexity !== 'number') { + handleBadRequest( + res, + next, + `JsonPath parameter '${paramName}' is invalid: contains disallowed operation '${astComplexity.disallowedOperation}'` + ); + } + + return jsonPathInput; +} + +/** + * Scan the a jsonpath expression to determine complexity. + * Disallow operations that could be used to perform expensive queries. + * See https://www.postgresql.org/docs/14/functions-json.html + */ +export function calculateJsonpathComplexity( + ast: JsonpathAst +): number | { disallowedOperation: string } { + let totalComplexity = 0; + const stack: JsonpathItem[] = [...ast.expr]; + + while (stack.length > 0) { + const item = stack.pop() as JsonpathItem; + + switch (item.type) { + // Recursive lookup operations not allowed + case '[*]': + case '.*': + case '.**': + // string "starts with" operation not allowed + case 'starts with': + // string regex operations not allowed + case 'like_regex': + // Index range operations not allowed + case 'last': + // Type coercion not allowed + case 'is_unknown': + // Item method operations not allowed + case 'type': + case 'size': + case 'double': + case 'ceiling': + case 'floor': + case 'abs': + case 'datetime': + case 'keyvalue': + return { disallowedOperation: item.type }; + + // Array index accessor + case '[subscript]': + if (item.elems.some(elem => elem.to.length > 0)) { + // Range operations not allowed + return { disallowedOperation: '[n to m] array range accessor' }; + } else { + totalComplexity += 1; + stack.push(...item.elems.flatMap(elem => elem.from)); + } + break; + + // Simple path navigation operations + case '$': + case '@': + break; + + // Path literals + case '$variable': + case '.key': + case 'null': + case 'string': + case 'numeric': + case 'bool': + totalComplexity += 1; + break; + + // Binary operations + case '&&': + case '||': + case '==': + case '!=': + case '<': + case '>': + case '<=': + case '>=': + case '+': + case '-': + case '*': + case '/': + case '%': + totalComplexity += 3; + stack.push(...item.left, ...item.right); + break; + + // Unary operations + case '?': + case '!': + case '+unary': + case '-unary': + case 'exists': + totalComplexity += 2; + stack.push(...item.arg); + break; + + default: + // @ts-expect-error - exhaustive switch + const unexpectedTypeID = item.type; + throw new Error(`Unexpected jsonpath expression type ID: ${unexpectedTypeID}`); + } + } + + return totalComplexity; +} + export function booleanValueForParam( req: Request, res: Response, diff --git a/src/api/routes/contract.ts b/src/api/routes/contract.ts index 531511ec0f..9be17cc76a 100644 --- a/src/api/routes/contract.ts +++ b/src/api/routes/contract.ts @@ -2,7 +2,7 @@ import * as express from 'express'; import { asyncHandler } from '../async-handler'; import { getPagingQueryLimit, parsePagingQueryInput, ResourceType } from '../pagination'; import { parseDbEvent } from '../controllers/db-controller'; -import { parseTraitAbi } from '../query-helpers'; +import { parseTraitAbi, validateJsonPathQuery } from '../query-helpers'; import { PgStore } from '../../datastore/pg-store'; export function createContractRouter(db: PgStore): express.Router { @@ -50,14 +50,44 @@ export function createContractRouter(db: PgStore): express.Router { router.get( '/:contract_id/events', - asyncHandler(async (req, res) => { + asyncHandler(async (req, res, next) => { const { contract_id } = req.params; const limit = getPagingQueryLimit(ResourceType.Contract, req.query.limit); const offset = parsePagingQueryInput(req.query.offset ?? 0); + + const filterPath = validateJsonPathQuery(req, res, next, 'filter_path', { + paramRequired: false, + maxCharLength: 200, + }); + + const containsJsonQuery = req.query['contains']; + if (containsJsonQuery && typeof containsJsonQuery !== 'string') { + res.status(400).json({ error: `'contains' query param must be a string` }); + return; + } + let containsJson: any | undefined; + const maxContainsJsonCharLength = 200; + if (containsJsonQuery) { + if (containsJsonQuery.length > maxContainsJsonCharLength) { + res.status(400).json({ + error: `'contains' query param value exceeds ${maxContainsJsonCharLength} character limit`, + }); + return; + } + try { + containsJson = JSON.parse(containsJsonQuery); + } catch (error) { + res.status(400).json({ error: `'contains' query param value must be valid JSON` }); + return; + } + } + const eventsQuery = await db.getSmartContractEvents({ contractId: contract_id, limit, offset, + filterPath, + containsJson, }); if (!eventsQuery.found) { res.status(404).json({ error: `cannot find events for contract by ID: ${contract_id}` }); diff --git a/src/datastore/common.ts b/src/datastore/common.ts index 3ff9b90a2c..fdff95becd 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -1379,6 +1379,7 @@ export interface SmartContractEventInsertValues { contract_identifier: string; topic: string; value: PgBytea; + value_json: string; } export interface BurnchainRewardInsertValues { diff --git a/src/datastore/migrations.ts b/src/datastore/migrations.ts index 931cc49f67..9b3f6a62c5 100644 --- a/src/datastore/migrations.ts +++ b/src/datastore/migrations.ts @@ -1,7 +1,16 @@ import * as path from 'path'; import PgMigrate, { RunnerOption } from 'node-pg-migrate'; -import { Client } from 'pg'; -import { APP_DIR, isDevEnv, isTestEnv, logError, logger, REPO_DIR } from '../helpers'; +import { Client, QueryResultRow } from 'pg'; +import * as PgCursor from 'pg-cursor'; +import { + APP_DIR, + clarityValueToCompactJson, + isDevEnv, + isTestEnv, + logError, + logger, + REPO_DIR, +} from '../helpers'; import { getPgClientConfig, PgClientConfig } from './connection-legacy'; import { connectPostgres, PgServer } from './connection'; import { databaseHasData } from './event-requests'; @@ -43,6 +52,7 @@ export async function runMigrations( runnerOpts.schema = clientConfig.schema; } await PgMigrate(runnerOpts); + await completeSqlMigrations(client, clientConfig); } catch (error) { logError(`Error running pg-migrate`, error); throw error; @@ -99,3 +109,98 @@ export async function dangerousDropAllTables(opts?: { await sql.end(); } } + +// Function to finish running sql migrations that are too complex for the node-pg-migrate library. +async function completeSqlMigrations(client: Client, clientConfig: PgClientConfig) { + try { + await client.query('BEGIN'); + await complete_1680181889941_contract_log_json(client, clientConfig); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} + +async function* pgCursorQuery(args: { + clientConfig: PgClientConfig; + queryText: string; + queryValues?: any[]; + batchSize: number; +}) { + const cursorClient = new Client(args.clientConfig); + try { + await cursorClient.connect(); + const cursor = new PgCursor(args.queryText, args.queryValues); + const cursorQuery = cursorClient.query(cursor); + let rows: R[] = []; + do { + rows = await new Promise((resolve, reject) => { + cursorQuery.read(args.batchSize, (error, rows) => (error ? reject(error) : resolve(rows))); + }); + yield* rows; + } while (rows.length > 0); + } finally { + await cursorClient.end(); + } +} + +async function complete_1680181889941_contract_log_json( + client: Client, + clientConfig: PgClientConfig +) { + // Determine if this migration has already been run by checking if the bew column is nullable. + const result = await client.query(` + SELECT is_nullable FROM information_schema.columns + WHERE table_name = 'contract_logs' AND column_name = 'value_json' + `); + const migrationNeeded = result.rows[0].is_nullable === 'YES'; + if (!migrationNeeded) { + return; + } + logger.info(`Running migration 1680181889941_contract_log_json..`); + + const contractLogsCursor = pgCursorQuery<{ id: string; value: string }>({ + clientConfig, + queryText: 'SELECT id, value FROM contract_logs', + batchSize: 1000, + }); + + const rowCountQuery = await client.query<{ count: number }>( + 'SELECT COUNT(*)::integer FROM contract_logs' + ); + const totalRowCount = rowCountQuery.rows[0].count; + let rowsProcessed = 0; + let lastPercentComplete = 0; + const percentLogInterval = 3; + + for await (const row of contractLogsCursor) { + const clarityValJson = JSON.stringify(clarityValueToCompactJson(row.value)); + await client.query({ + name: 'update_contract_log_json', + text: 'UPDATE contract_logs SET value_json = $1 WHERE id = $2', + values: [clarityValJson, row.id], + }); + rowsProcessed++; + const percentComplete = Math.round((rowsProcessed / totalRowCount) * 100); + if (percentComplete > lastPercentComplete + percentLogInterval) { + lastPercentComplete = percentComplete; + logger.info(`Running migration 1680181889941_contract_log_json.. ${percentComplete}%`); + } + } + + logger.info(`Running migration 1680181889941_contract_log_json.. set NOT NULL`); + await client.query(`ALTER TABLE contract_logs ALTER COLUMN value_json SET NOT NULL`); + + logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_path_ops index'); + await client.query( + `CREATE INDEX contract_logs_jsonpathops_idx ON contract_logs USING GIN (value_json jsonb_path_ops)` + ); + + logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_ops index'); + await client.query( + `CREATE INDEX contract_logs_jsonops_idx ON contract_logs USING GIN (value_json jsonb_ops)` + ); + + logger.info(`Running migration 1680181889941_contract_log_json.. 100%`); +} diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 1afc8539de..17848c27df 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -2094,11 +2094,18 @@ export class PgStore { contractId, limit, offset, + filterPath, + containsJson, }: { contractId: string; limit: number; offset: number; + filterPath: string | null; + containsJson: any | undefined; }): Promise> { + const hasFilterPath = filterPath !== null; + const hasJsonContains = containsJson !== undefined; + const logResults = await this.sql< { event_index: number; @@ -2108,12 +2115,15 @@ export class PgStore { contract_identifier: string; topic: string; value: string; + value_json: any; }[] >` SELECT event_index, tx_id, tx_index, block_height, contract_identifier, topic, value FROM contract_logs WHERE canonical = true AND microblock_canonical = true AND contract_identifier = ${contractId} + ${hasFilterPath ? this.sql`AND value_json @? ${filterPath}::jsonpath` : this.sql``} + ${hasJsonContains ? this.sql`AND value_json @> ${containsJson}::jsonb` : this.sql``} ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC LIMIT ${limit} OFFSET ${offset} diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index b0d9fe9ed5..c504f8fa1c 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -1,4 +1,12 @@ -import { logger, logError, getOrAdd, batchIterate, isProdEnv, I32_MAX } from '../helpers'; +import { + logger, + logError, + getOrAdd, + batchIterate, + isProdEnv, + I32_MAX, + clarityValueToCompactJson, +} from '../helpers'; import { DbBlock, DbTx, @@ -1228,6 +1236,7 @@ export class PgWriteStore extends PgStore { contract_identifier: event.contract_identifier, topic: event.topic, value: event.value, + value_json: JSON.stringify(clarityValueToCompactJson(event.value)), })); const res = await sql` INSERT INTO contract_logs ${sql(values)} @@ -1253,6 +1262,7 @@ export class PgWriteStore extends PgStore { contract_identifier: event.contract_identifier, topic: event.topic, value: event.value, + value_json: JSON.stringify(clarityValueToCompactJson(event.value)), }; await sql` INSERT INTO contract_logs ${sql(values)} diff --git a/src/helpers.ts b/src/helpers.ts index 77316307fa..62d38ffe29 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -14,7 +14,13 @@ import * as dotenv from 'dotenv-flow'; import * as http from 'http'; import { isArrayBufferView } from 'node:util/types'; import * as path from 'path'; -import { isValidStacksAddress, stacksToBitcoinAddress } from 'stacks-encoding-native-js'; +import { + ClarityTypeID, + ClarityValue, + decodeClarityValue, + isValidStacksAddress, + stacksToBitcoinAddress, +} from 'stacks-encoding-native-js'; import * as stream from 'stream'; import * as ecc from 'tiny-secp256k1'; import * as util from 'util'; @@ -327,6 +333,71 @@ export function isValidPrincipal( return false; } +/** + * Encodes a Clarity value into a JSON object. This encoding does _not_ preserve exact Clarity type information. + * Instead, values are mapped to a JSON object that is optimized for readability and interoperability with + * JSON-based workflows (e.g. Postgres `jsonpath` support). + * * OptionalSome and ResponseOk are unwrapped (i.e. the value is encoded directly without any nesting). + * * OptionalNone is encoded as json `null`. + * * ResponseError is encoded as an object with a single key `_error`. + * * Buffers are encoded as an object containing the key `hex` with the hex-encoded string as the value, + * and the key `utf8` with the utf8-encoded string as the value. When decoding a Buffer into a string that + * does not exclusively contain valid UTF-8 data, the Unicode replacement character U+FFFD `�` will be used + * to represent those errors. + * * Ints and UInts that are in the range of a safe js integers are encoded as numbers, otherwise they + * are encoded as string-quoted integers. + * * Booleans are encoded as booleans. + * * Principals are encoded as strings, e.g. `
` or `
.`. + * * StringAscii and StringUtf8 are both encoded as regular json strings. + * * Lists are encoded as json arrays. + * * Tuples are encoded as json objects. + * @param cv - the Clarity value to encode, or a hex-encoded string representation of a Clarity value. + */ +export function clarityValueToCompactJson(clarityValue: ClarityValue | string): any { + let cv: ClarityValue; + if (typeof clarityValue === 'string') { + cv = decodeClarityValue(clarityValue); + } else { + cv = clarityValue; + } + switch (cv.type_id) { + case ClarityTypeID.Int: + case ClarityTypeID.UInt: + const intVal = parseInt(cv.value); + return Number.isSafeInteger(intVal) ? intVal : cv.value; + case ClarityTypeID.BoolTrue: + case ClarityTypeID.BoolFalse: + return cv.value; + case ClarityTypeID.StringAscii: + case ClarityTypeID.StringUtf8: + return cv.data; + case ClarityTypeID.ResponseOk: + case ClarityTypeID.OptionalSome: + return clarityValueToCompactJson(cv.value); + case ClarityTypeID.PrincipalStandard: + return cv.address; + case ClarityTypeID.PrincipalContract: + return cv.address + '.' + cv.contract_name; + case ClarityTypeID.ResponseError: + return { _error: clarityValueToCompactJson(cv.value) }; + case ClarityTypeID.OptionalNone: + return null; + case ClarityTypeID.List: + return cv.list.map(clarityValueToCompactJson) as any; + case ClarityTypeID.Tuple: + return Object.fromEntries( + Object.entries(cv.data).map(([key, value]) => [key, clarityValueToCompactJson(value)]) + ); + case ClarityTypeID.Buffer: + return { + hex: cv.buffer, + utf8: Buffer.from(cv.buffer.substring(2), 'hex').toString('utf8'), + }; + } + // @ts-expect-error - all ClarityTypeID cases are handled above + throw new Error(`Unexpected Clarity type ID: ${cv.type_id}`); +} + export type HttpClientResponse = http.IncomingMessage & { statusCode: number; statusMessage: string; diff --git a/src/tests/cv-compact-json-encoding-tests.ts b/src/tests/cv-compact-json-encoding-tests.ts new file mode 100644 index 0000000000..0ae6a13473 --- /dev/null +++ b/src/tests/cv-compact-json-encoding-tests.ts @@ -0,0 +1,89 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable prettier/prettier */ +import { clarityValueToCompactJson } from '../helpers'; +import { decodeClarityValueToRepr } from 'stacks-encoding-native-js'; + +describe('clarity value compact json encoding', () => { + + const clarityCompactJsonVectors: [repr: string, hexEncodedCV: string, compactJson: string][] = [ + [ + `(tuple (action "voted") (data (tuple (contract 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda) (end-block-height u98088) (is-ended false) (no-votes u0) (proposer 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5) (start-block-height u97080) (title u"xUSD/USDA Farming at Arkadiko") (url u"https://discord.com/channels/923585641554518026/934026731256422450") (yes-votes u14965111242))) (type "proposal"))`, + '0x0c0000000306616374696f6e0d00000005766f74656404646174610c0000000908636f6e74726163740616e8be428cdb133f95ba579f4805a053a4cc1eaeae1770726f706f73616c2d6661726d2d787573642d7573646110656e642d626c6f636b2d6865696768740100000000000000000000000000017f280869732d656e64656404086e6f2d766f74657301000000000000000000000000000000000870726f706f7365720516e8be428cdb133f95ba579f4805a053a4cc1eaeae1273746172742d626c6f636b2d6865696768740100000000000000000000000000017b38057469746c650e0000001d785553442f55534441204661726d696e672061742041726b6164696b6f0375726c0e0000004268747470733a2f2f646973636f72642e636f6d2f6368616e6e656c732f3932333538353634313535343531383032362f393334303236373331323536343232343530097965732d766f746573010000000000000000000000037bfd79ca04747970650d0000000870726f706f73616c', + '{"action":"voted","data":{"contract":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda","end-block-height":98088,"is-ended":false,"no-votes":0,"proposer":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5","start-block-height":97080,"title":"xUSD/USDA Farming at Arkadiko","url":"https://discord.com/channels/923585641554518026/934026731256422450","yes-votes":14965111242},"type":"proposal"}' + ], + [ + `(tuple (action "accept-bid") (payload (tuple (action_event_index u1) (bid_amount u22000000) (bid_id u4572) (bidder_address 'SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8) (collection_id 'SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh) (expiration_block u97322) (royalty (tuple (percent u250) (recipient_address 'SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T))) (seller_address 'SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66) (token_id u907))))`, + '0x0c0000000206616374696f6e0d0000000a6163636570742d626964077061796c6f61640c0000000912616374696f6e5f6576656e745f696e64657801000000000000000000000000000000010a6269645f616d6f756e7401000000000000000000000000014fb180066269645f696401000000000000000000000000000011dc0e6269646465725f6164647265737305166596aa0af759a86e95d347116236477da7e137400d636f6c6c656374696f6e5f6964061625c0f5c23a3d463bd4ead1f7c2548a3fb529cdb00b68617070792d77656c73681065787069726174696f6e5f626c6f636b0100000000000000000000000000017c2a07726f79616c74790c000000020770657263656e7401000000000000000000000000000000fa11726563697069656e745f61646472657373051691c23287d1b1e8762d50177b2fd59fb3acbd96050e73656c6c65725f6164647265737305166b0a653f222d8ebd501526bcd816e3b0f01d809e08746f6b656e5f6964010000000000000000000000000000038b', + '{"action":"accept-bid","payload":{"action_event_index":1,"bid_amount":22000000,"bid_id":4572,"bidder_address":"SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8","collection_id":"SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh","expiration_block":97322,"royalty":{"percent":250,"recipient_address":"SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T"},"seller_address":"SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66","token_id":907}}' + ], + [ + `(tuple (action "swap-x-for-y") (data (tuple (balance-x u315175871745570) (balance-y u11937495544) (fee-rate-x u300000) (fee-rate-y u300000) (fee-rebate u50000000) (fee-to-address 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01) (oracle-average u95000000) (oracle-enabled true) (oracle-resilient u3871) (pool-token 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01) (total-supply u3290909475111))) (object "pool"))`, + '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c0000000b0962616c616e63652d7801000000000000000000011ea699e08e220962616c616e63652d7901000000000000000000000002c787b9f80a6665652d726174652d7801000000000000000000000000000493e00a6665652d726174652d7901000000000000000000000000000493e00a6665652d7265626174650100000000000000000000000002faf0800e6665652d746f2d616464726573730616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d226d756c74697369672d6677702d777374782d776274632d35302d35302d76312d30310e6f7261636c652d617665726167650100000000000000000000000005a995c00e6f7261636c652d656e61626c656403106f7261636c652d726573696c69656e740100000000000000000000000000000f1f0a706f6f6c2d746f6b656e0616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d196677702d777374782d776274632d35302d35302d76312d30310c746f74616c2d737570706c79010000000000000000000002fe397d8127066f626a6563740d00000004706f6f6c', + '{"action":"swap-x-for-y","data":{"balance-x":315175871745570,"balance-y":11937495544,"fee-rate-x":300000,"fee-rate-y":300000,"fee-rebate":50000000,"fee-to-address":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01","oracle-average":95000000,"oracle-enabled":true,"oracle-resilient":3871,"pool-token":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01","total-supply":3290909475111},"object":"pool"}' + ], + [ + `(tuple (action "swap-x-for-y") (data (tuple (balance-x u39248987199) (balance-y u294901858687020) (enabled true) (fee-balance-x u209257577) (fee-balance-y u3188281977630) (fee-to-address (some 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR)) (name "wSTX-WELSH") (shares-total u3135066566725) (swap-token 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh))) (object "pair"))`, + '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c000000090962616c616e63652d7801000000000000000000000009236c043f0962616c616e63652d7901000000000000000000010c363087d82c07656e61626c6564030d6665652d62616c616e63652d78010000000000000000000000000c7904690d6665652d62616c616e63652d79010000000000000000000002e6546a2b1e0e6665652d746f2d616464726573730a0516982f3ec112a5f5928a5c96a914bd733793b896a5046e616d650d0000000a775354582d57454c53480c7368617265732d746f74616c010000000000000000000002d9f08770450a737761702d746f6b656e0616982f3ec112a5f5928a5c96a914bd733793b896a51e61726b6164696b6f2d737761702d746f6b656e2d777374782d77656c7368066f626a6563740d0000000470616972', + '{"action":"swap-x-for-y","data":{"balance-x":39248987199,"balance-y":294901858687020,"enabled":true,"fee-balance-x":209257577,"fee-balance-y":3188281977630,"fee-to-address":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR","name":"wSTX-WELSH","shares-total":3135066566725,"swap-token":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh"},"object":"pair"}' + ], + [ + `(tuple (attachment (tuple (attachment-index u360918) (hash 0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236) (metadata (tuple (name 0x64657369676e6d616e6167656d656e74636f6e73756c74696e67) (namespace 0x627463) (op "name-register") (tx-sender 'SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9))))))`, + '0x0c000000010a6174746163686d656e740c00000003106174746163686d656e742d696e64657801000000000000000000000000000581d604686173680200000014ffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236086d657461646174610c00000004046e616d65020000001a64657369676e6d616e6167656d656e74636f6e73756c74696e67096e616d6573706163650200000003627463026f700d0000000d6e616d652d72656769737465720974782d73656e6465720516e511302be0f5e10d5c2f422d596fa67f1d6c1a5f', + '{"attachment":{"attachment-index":360918,"hash":{"hex":"0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236","utf8":"���\\u000f�������\\u00050�̓���6"},"metadata":{"name":{"hex":"0x64657369676e6d616e6167656d656e74636f6e73756c74696e67","utf8":"designmanagementconsulting"},"namespace":{"hex":"0x627463","utf8":"btc"},"op":"name-register","tx-sender":"SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9"}}}' + ], + [ + `(tuple (amountStacked u7420000000000) (cityId u1) (cityName "mia") (cityTreasury 'SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking) (event "stacking") (firstCycle u54) (lastCycle u59) (lockPeriod u6) (userId u136))`, + '0x0c000000090d616d6f756e74537461636b6564010000000000000000000006bf9a76d80006636974794964010000000000000000000000000000000108636974794e616d650d000000036d69610c636974795472656173757279061610a4c7e3b4f3a06482dd12510fe9aec82cfc02361c6363643030322d74726561737572792d6d69612d737461636b696e67056576656e740d00000008737461636b696e670a66697273744379636c650100000000000000000000000000000036096c6173744379636c65010000000000000000000000000000003b0a6c6f636b506572696f640100000000000000000000000000000006067573657249640100000000000000000000000000000088', + '{"amountStacked":7420000000000,"cityId":1,"cityName":"mia","cityTreasury":"SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking","event":"stacking","firstCycle":54,"lastCycle":59,"lockPeriod":6,"userId":136}' + ], + [ + '0x3231333431363630323537', + '0x020000000b3231333431363630323537', + '{"hex":"0x3231333431363630323537","utf8":"21341660257"}' + ], + [ + 'none', + '0x09', + 'null' + ], + [ + '"``... to be a completely separate network and separate block chain, yet share CPU power with Bitcoin`` - Satoshi Nakamoto"', + '0x0d0000007960602e2e2e20746f206265206120636f6d706c6574656c79207365706172617465206e6574776f726b20616e6420736570617261746520626c6f636b20636861696e2c207965742073686172652043505520706f776572207769746820426974636f696e6060202d205361746f736869204e616b616d6f746f', + '"``... to be a completely separate network and separate block chain, yet share CPU power with Bitcoin`` - Satoshi Nakamoto"' + ], + [ + "'SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60", + '0x0516661506d48705f932af21abcff23046b216886e84', + '"SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60"' + ], + [ + '"Is this thing on?"', + '0x0d0000001149732074686973207468696e67206f6e3f', + '"Is this thing on?"' + ], + [ + '0x324d775773533233676638346554', + '0x020000000e324d775773533233676638346554', + '{"hex":"0x324d775773533233676638346554","utf8":"2MwWsS23gf84eT"}' + ] + ]; + + test.each(clarityCompactJsonVectors)('test Clarity value compact json encoding: %p', (reprString, hexEncodedCV, compactJson) => { + const encodedJson = clarityValueToCompactJson(hexEncodedCV); + const reprResult = decodeClarityValueToRepr(hexEncodedCV); + expect(encodedJson).toEqual(JSON.parse(compactJson)); + expect(reprResult).toBe(reprString); + }); + + /* + test.skip('generate', () => { + const result = clarityCompactJsonVectors.map(([, hexEncodedCV, compactJson]) => { + const encodedJson = clarityValueToCompactJson(hexEncodedCV); + const reprResult = decodeClarityValueToRepr(hexEncodedCV); + return [reprResult, hexEncodedCV, JSON.stringify(encodedJson)]; + }); + console.log(result); + }); + */ + +}); diff --git a/src/tests/jsonpath-tests.ts b/src/tests/jsonpath-tests.ts new file mode 100644 index 0000000000..2d3c6a667c --- /dev/null +++ b/src/tests/jsonpath-tests.ts @@ -0,0 +1,60 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable prettier/prettier */ +import jsonpathToAst from 'jsonpath-pg'; +import { calculateJsonpathComplexity } from '../api/query-helpers'; + +describe('jsonpath tests', () => { + + const jsonPathVectors: [string, string | null][] = [ + [ '$.track.segments.location', null ], + [ '$[0] + 3', null ], + [ '+ $.x', null ], + [ '7 - $[0]', null ], + [ '- $.x', null ], + [ '2 * $[0]', null ], + [ '$[0] / 2', null ], + [ '$[0] % 10', null ], + [ '$[1,3,4,6]', null ], + [ '$[555 to LAST]', '[n to m] array range accessor' ], + [ '$[1 to 37634].a', '[n to m] array range accessor' ], + [ '$[3,4 to last].a', '[n to m] array range accessor' ], + [ '$[(3 + 4) to last].a', '[n to m] array range accessor' ], + [ '$[1 to (3 + 4)].a', '[n to m] array range accessor' ], + [ '$.**.HR', '.**' ], + [ '$.* ? (@.v == "a")', '.*' ], + [ '$.** ? (@.v == "a")', '.**' ], + [ '$.track.segments[*].location', '[*]' ], + [ '$.t.type()', 'type' ], + [ '$.m.size()', 'size' ], + [ '$.len.double() * 2', 'double' ], + [ '$.h.ceiling()', 'ceiling' ], + [ '$.h.floor()', 'floor' ], + [ '$.z.abs()', 'abs' ], + [ '$.a ? (@.datetime() < "2015-08-2".datetime())', 'datetime' ], + [ '$.a.datetime("HH24:MI")', 'datetime' ], + [ '$.keyvalue()', 'keyvalue' ], + [ '$.a ? (@ like_regex "^abc")', 'like_regex' ], + [ '$.a ? (@ starts with "John")', 'starts with' ], + [ '$.a ? ((@ > 0) is unknown)', 'is_unknown' ] + ]; + + test.each(jsonPathVectors)('test jsonpath operation complexity: %p', (input, result) => { + const ast = jsonpathToAst(input); + const disallowed = calculateJsonpathComplexity(ast); + if (typeof disallowed === 'number') { + expect(result).toBe(null); + } else { + expect(disallowed.disallowedOperation).toBe(result); + } + }); + + test.skip('generate vector data', () => { + const result = jsonPathVectors.map(([input]) => { + const ast = jsonpathToAst(input); + const complexity = calculateJsonpathComplexity(ast); + return [input, typeof complexity !== 'number' ? complexity.disallowedOperation : null]; + }); + console.log(result); + }); + +});