Skip to content

feat: added querying historical data by block range #2906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: added querying historical data by block range #2906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions packages/query/src/graphql/__snapshots__/graphql.blockrange.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`GraphqlBlockRange should filter entities by block range 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name"), 'value'::text, ((__local_1__."value"))::text))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlBlockRange should handle empty block range gracefully 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name")))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlBlockRange should include __block_height in SQL for result transformation 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name")))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlBlockRange should return single entity with block range filter 1`] = `
"select
to_json(lower(__local_0__._block_range)) as "__block_height", to_json((__local_0__."id")) as "id", to_json((__local_0__."name")) as "name", to_json(((__local_0__."value"))::text) as "value"
from "subquery_blockrange_test"."test_entities" as __local_0__

where (__local_0__."id" = $1) and (__local_0__._block_range && int8range($2::bigint, $3::bigint, '[]')) and (TRUE) and (TRUE)


"
`;

exports[`GraphqlBlockRange should validate blockRange parameter format 1`] = `
"with __local_0__ as (select to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name")))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlBlockRange should work with existing blockHeight parameter (backwards compatibility) 1`] = `
"with __local_0__ as (select to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name")))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (__local_1__._block_range @> $1::bigint) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlBlockRange should work with filtering and block range together 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'name'::text, (__local_1__."name"), 'value'::text, ((__local_1__."value"))::text))) as "@nodes" from (select
__local_1__.*
from "subquery_blockrange_test"."test_entities" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (((__local_1__."name" LIKE $3))) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;
80 changes: 71 additions & 9 deletions packages/query/src/graphql/__snapshots__/graphql.historical.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,80 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`GraphqlHistorical should generate block range SQL for entity queries 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'priceAmount'::text, ((__local_1__."price_amount"))::text))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."listings" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlHistorical should include blockRange parameter in schema for collections 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'ownerId'::text, (__local_1__."owner_id")))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlHistorical should maintain backwards compatibility with existing blockHeight 1`] = `
"with __local_0__ as (select to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'ownerId'::text, (__local_1__."owner_id")))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range @> $1::bigint) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlHistorical should support block range in nested relations 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), '@listings'::text, (with __local_2__ as (select to_json(lower(__local_3__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_3__."_id"), 'id'::text, (__local_3__."id"), 'priceAmount'::text, ((__local_3__."price_amount"))::text))) as "@nodes" from (select
__local_3__.*
from "subquery_2"."listings" as __local_3__

where (__local_3__."item_id" = __local_1__."id") and (__local_3__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (__local_3__._block_range && int8range($3::bigint, $4::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_3__."_id" ASC

) __local_3__), __local_4__ as (select json_agg(to_json(__local_2__)) as data from __local_2__) select json_build_object('data'::text, coalesce((select __local_4__.data from __local_4__), '[]'::json)) )))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range && int8range($5::bigint, $6::bigint, '[]')) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_5__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_5__.data from __local_5__), '[]'::json) as "data" "
`;

exports[`GraphqlHistorical should work with block range and filtering together 1`] = `
"with __local_0__ as (select to_json(lower(__local_1__._block_range)) as "__block_height", to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), 'id'::text, (__local_1__."id"), 'approved'::text, (__local_1__."approved")))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range && int8range($1::bigint, $2::bigint, '[]')) and (((__local_1__."approved" = $3))) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_2__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_2__.data from __local_2__), '[]'::json) as "data" "
`;

exports[`GraphqlHistorical to filter historical items when ordering 1`] = `
"with __local_0__ as (select to_json((json_build_object('__identifiers'::text, json_build_array(__local_1__."_id"), '@listings'::text, (with __local_2__ as (select to_json((json_build_object('__identifiers'::text, json_build_array(__local_3__."_id"), 'priceAmount'::text, ((__local_3__."price_amount"))::text))) as "@nodes" from (select
__local_3__.*
from "subquery_2"."listings" as __local_3__

where (__local_3__."item_id" = __local_1__."id") and (__local_3__._block_range @> $1::bigint) and (__local_3__._block_range @> $1::bigint) and (TRUE) and (TRUE)
where (__local_3__."item_id" = __local_1__."id") and (TRUE) and (TRUE)
order by __local_3__."_id" ASC

) __local_3__), __local_4__ as (select json_agg(to_json(__local_2__)) as data from __local_2__) select json_build_object('data'::text, coalesce((select __local_4__.data from __local_4__), '[]'::json)) )))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range @> $1::bigint) and (TRUE) and (TRUE)
where (TRUE) and (TRUE)
order by __local_1__."last_traded_price_amount" ASC,__local_1__."_id" ASC

) __local_1__), __local_5__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_5__.data from __local_5__), '[]'::json) as "data" "
Expand All @@ -23,7 +85,7 @@ exports[`GraphqlHistorical to filter historical nested (backward) 1`] = `
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range @> $1::bigint) and (((exists(select 1 from "subquery_2"."listings" as __local_2__ where (__local_2__."item_id" = __local_1__."id") and (__local_2__._block_range @> $1::bigint) and (((__local_2__."price_token" = $2))) and (__local_2__._block_range @> $1::bigint))))) and (TRUE) and (TRUE)
where (((exists(select 1 from "subquery_2"."listings" as __local_2__ where (__local_2__."item_id" = __local_1__."id") and (((__local_2__."price_token" = $1))))))) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_3__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_3__.data from __local_3__), '[]'::json) as "data" "
Expand All @@ -34,10 +96,10 @@ exports[`GraphqlHistorical to filter historical nested (forward) 1`] = `
__local_1__.*
from "subquery_2"."listings" as __local_1__

where (__local_1__._block_range @> $1::bigint) and (( exists(
where (( exists(
select 1 from "subquery_2"."items" as __local_2__
where (__local_1__."item_id" = __local_2__."id") and (__local_1__._block_range @> $1::bigint) and
(((__local_2__."approved" = $2))) and (__local_2__._block_range @> $1::bigint)
where (__local_1__."item_id" = __local_2__."id") and
(((__local_2__."approved" = $1)))
))) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

Expand All @@ -49,19 +111,19 @@ exports[`GraphqlHistorical to filter historical top level 1`] = `
__local_3__.*
from "subquery_2"."listings" as __local_3__

where (__local_3__."item_id" = __local_1__."id") and (__local_3__._block_range @> $1::bigint) and (__local_3__._block_range @> $1::bigint) and (TRUE) and (TRUE)
where (__local_3__."item_id" = __local_1__."id") and (TRUE) and (TRUE)
order by __local_3__."_id" ASC

) __local_3__), __local_4__ as (select json_agg(to_json(__local_2__)) as data from __local_2__) select json_build_object('data'::text, coalesce((select __local_4__.data from __local_4__), '[]'::json)) )))) as "@nodes" from (select
__local_1__.*
from "subquery_2"."items" as __local_1__

where (__local_1__._block_range @> $1::bigint) and ((exists(select 1 from "subquery_2"."listings" as __local_5__ where (__local_5__."item_id" = __local_1__."id") and (__local_5__._block_range @> $1::bigint)))) and (TRUE) and (TRUE)
where ((exists(select 1 from "subquery_2"."listings" as __local_5__ where (__local_5__."item_id" = __local_1__."id")))) and (TRUE) and (TRUE)
order by __local_1__."_id" ASC

) __local_1__), __local_6__ as (select json_agg(to_json(__local_0__)) as data from __local_0__) select coalesce((select __local_6__.data from __local_6__), '[]'::json) as "data", (
select json_build_object('totalCount'::text, count(1))
from "subquery_2"."items" as __local_1__
where (__local_1__._block_range @> $1::bigint) and ((exists(select 1 from "subquery_2"."listings" as __local_5__ where (__local_5__."item_id" = __local_1__."id") and (__local_5__._block_range @> $1::bigint))))
where ((exists(select 1 from "subquery_2"."listings" as __local_5__ where (__local_5__."item_id" = __local_1__."id"))))
) as "aggregates" "
`;
188 changes: 188 additions & 0 deletions packages/query/src/graphql/graphql.blockrange.compatibility.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
// Copyright 2020-2025 SubQuery Pte Ltd authors & contributors
// SPDX-License-Identifier: GPL-3.0

import {getPostGraphileBuilder} from '@subql/x-postgraphile-core';
import {ApolloServer, ExpressContext, gql} from 'apollo-server-express';
import {Pool} from 'pg';
import {Config} from '../configure';
import {getYargsOption} from '../yargs';
import {plugins} from './plugins';

jest.mock('../yargs', () => jest.createMockFromModule('../yargs'));

(getYargsOption as jest.Mock).mockImplementation(() => {
return {argv: {name: 'test', aggregate: true}};
});

describe('GraphqlBlockRange - Backwards Compatibility', () => {
const dbSchema = 'subquery_compatibility_test';

const config = new Config({});

const pool: Pool = new Pool({
user: config.get('DB_USER'),
password: config.get('DB_PASS'),
host: config.get('DB_HOST_READ') ?? config.get('DB_HOST'),
port: config.get('DB_PORT'),
database: config.get('DB_DATABASE'),
});

pool.on('error', (err) => {
console.error('PostgreSQL client generated error: ', err.message);
});

let server: ApolloServer<ExpressContext>;
let sqlSpy: jest.SpyInstance<void, [queryText: string, values: any[], callback?: any]>;

async function createApolloServer() {
const builder = await getPostGraphileBuilder(pool, [dbSchema], {
replaceAllPlugins: plugins,
subscriptions: true,
dynamicJson: true,
});

const schema = builder.buildSchema();

const server = new ApolloServer({
schema,
context: {
pgClient: pool,
},
});

return server;
}

beforeAll(async () => {
await pool.query(`CREATE SCHEMA IF NOT EXISTS ${dbSchema}`);

await pool.query(`CREATE TABLE IF NOT EXISTS "${dbSchema}".entities (
id text NOT NULL,
name text NOT NULL,
value numeric NOT NULL,
created_at_block_height numeric NOT NULL,
updated_at_block_height numeric NULL,
"_id" uuid NOT NULL,
"_block_range" int8range NOT NULL,
CONSTRAINT entities_pkey PRIMARY KEY (_id)
);`);

await pool.query(`INSERT INTO "${dbSchema}".entities
(id, name, value, created_at_block_height, "_id", "_block_range") VALUES
('entity1', 'Current Version', 100, 10, gen_random_uuid(), int8range(10, null)),
('entity2', 'Another Entity', 200, 15, gen_random_uuid(), int8range(15, null));`);

server = await createApolloServer();
sqlSpy = jest.spyOn(pool, 'query');
});

beforeEach(() => {
sqlSpy.mockClear();
});

afterAll(async () => {
await pool.query(`DROP TABLE IF EXISTS "${dbSchema}".entities;`);
await pool.query(`DROP SCHEMA IF EXISTS ${dbSchema};`);
await pool.end();
});

it('should still work with existing blockHeight queries', async () => {
const GQL_QUERY = gql`
query entitiesByBlockHeight {
entities(blockHeight: "15") {
nodes {
id
name
value
}
}
}
`;

const res = await server.executeOperation({query: GQL_QUERY});
expect(res.errors).toBeUndefined();

expect(sqlSpy.mock.calls[0][0]).toContain('_block_range @>');
expect(sqlSpy.mock.calls[0][0]).not.toContain('&&');
expect(sqlSpy.mock.calls[0][0]).not.toContain('int8range');
});

it('should prevent using blockRange with blockHeight together', async () => {
const GQL_QUERY = gql`
query entitiesWithConflictingParams {
entities(blockHeight: "15", blockRange: ["10", "20"]) {
nodes {
id
name
}
}
}
`;

const res = await server.executeOperation({query: GQL_QUERY});
expect(res.errors).toBeUndefined();

expect(sqlSpy.mock.calls[0][0]).not.toContain('_block_range @>');
expect(sqlSpy.mock.calls[0][0]).not.toContain('&&');

expect(res.data?.entities?.nodes).toBeDefined();
});

it('should validate blockRange parameters', async () => {
const GQL_QUERY = gql`
query entitiesWithInvalidRange {
entities(blockRange: ["20", "10"]) {
nodes {
id
name
}
}
}
`;

const res = await server.executeOperation({query: GQL_QUERY});
expect(res.errors || sqlSpy.mock.calls[0]?.[0]).toBeDefined();
});

it('should maintain existing query performance characteristics', async () => {
const GQL_QUERY = gql`
query entitiesPerformanceTest {
entities(blockHeight: "15", first: 10) {
nodes {
id
name
}
totalCount
}
}
`;

const res = await server.executeOperation({query: GQL_QUERY});
expect(res.errors).toBeUndefined();

// Should include pagination and aggregation
expect(sqlSpy.mock.calls[0][0]).toContain('_block_range @>');
expect(res.data?.entities?.totalCount).toBeDefined();
});

it('should work with existing filtering and relations', async () => {
const GQL_QUERY = gql`
query entitiesWithFiltering {
entities(blockHeight: "15", filter: {name: {includes: "Entity"}}) {
nodes {
id
name
value
}
}
}
`;

const res = await server.executeOperation({query: GQL_QUERY});
expect(res.errors).toBeUndefined();

// Should combine block height with other filters
expect(sqlSpy.mock.calls[0][0]).toContain('_block_range @>');
expect(sqlSpy.mock.calls[0][0]).toContain('name');
});
});
Loading