Skip to content

Commit 94b6c86

Browse files
Maddiaa0Maddiaa0
authored
feat(blobs): blob sink (#10079)
fixes: #10053
1 parent 5e3183c commit 94b6c86

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1288
-60
lines changed

‎yarn-project/blob-sink/.eslintrc.cjs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
module.exports = require('@aztec/foundation/eslint');

‎yarn-project/blob-sink/README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
## Blob Sink
2+
3+
A HTTP api that losely emulates the https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars API.
4+
We do not support all of the possible values of block_id, namely `genesis`, `head`, `finalized`. As we are not using any of these values in our
5+
blobs integration.
6+
7+
## When is this used?
8+
9+
This service will run alongside end to end tests to capture the blob transactions that are sent alongside a `propose` transaction.
10+
11+
### Why?
12+
13+
Once we make the transition to blob transactions, we will need to be able to query for blobs. One way to do this is to run an entire L1 execution layer and consensus layer pair alongside all of our e2e tests and inside the sandbox. But this is a bit much, so instead the blob sink can be used to store and request blobs, without needing to run an entire consensus layer pair client.
14+
15+
### Other Usecases
16+
17+
Blobs are only held in the L1 consensus layer for a period of ~3 weeks, the blob sink can be used to store blobs for longer.
18+
19+
### How?
20+
21+
The blob sink is a simple HTTP server that can be run alongside the e2e tests. It will store the blobs in a local file system and provide an API to query for them.

‎yarn-project/blob-sink/package.json

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"name": "@aztec/blob-sink",
3+
"version": "0.1.0",
4+
"type": "module",
5+
"exports": {
6+
".": "./dest/index.js"
7+
},
8+
"inherits": [
9+
"../package.common.json"
10+
],
11+
"scripts": {
12+
"build": "yarn clean && tsc -b",
13+
"build:dev": "tsc -b --watch",
14+
"clean": "rm -rf ./dest .tsbuildinfo",
15+
"formatting": "run -T prettier --check ./src && run -T eslint ./src",
16+
"formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
17+
"test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
18+
},
19+
"jest": {
20+
"moduleNameMapper": {
21+
"^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
22+
},
23+
"testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
24+
"rootDir": "./src",
25+
"transform": {
26+
"^.+\\.tsx?$": [
27+
"@swc/jest",
28+
{
29+
"jsc": {
30+
"parser": {
31+
"syntax": "typescript",
32+
"decorators": true
33+
},
34+
"transform": {
35+
"decoratorVersion": "2022-03"
36+
}
37+
}
38+
}
39+
]
40+
},
41+
"extensionsToTreatAsEsm": [
42+
".ts"
43+
],
44+
"reporters": [
45+
"default"
46+
],
47+
"testTimeout": 30000,
48+
"setupFiles": [
49+
"../../foundation/src/jest/setup.mjs"
50+
]
51+
},
52+
"dependencies": {
53+
"@aztec/circuit-types": "workspace:^",
54+
"@aztec/foundation": "workspace:^",
55+
"@aztec/kv-store": "workspace:*",
56+
"@aztec/telemetry-client": "workspace:*",
57+
"express": "^4.21.1",
58+
"source-map-support": "^0.5.21",
59+
"tslib": "^2.4.0",
60+
"zod": "^3.23.8"
61+
},
62+
"devDependencies": {
63+
"@jest/globals": "^29.5.0",
64+
"@types/jest": "^29.5.0",
65+
"@types/memdown": "^3.0.0",
66+
"@types/node": "^18.7.23",
67+
"@types/source-map-support": "^0.5.10",
68+
"@types/supertest": "^6.0.2",
69+
"jest": "^29.5.0",
70+
"jest-mock-extended": "^3.0.3",
71+
"supertest": "^7.0.0",
72+
"ts-node": "^10.9.1",
73+
"typescript": "^5.0.4"
74+
},
75+
"files": [
76+
"dest",
77+
"src",
78+
"!*.test.*"
79+
],
80+
"types": "./dest/index.d.ts",
81+
"engines": {
82+
"node": ">=18"
83+
}
84+
}

‎yarn-project/blob-sink/src/blob-sink.test.ts

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { Blob } from '@aztec/foundation/blob';
2+
import { Fr } from '@aztec/foundation/fields';
3+
4+
import request from 'supertest';
5+
6+
import { BlobSinkServer } from './server.js';
7+
8+
describe('BlobSinkService', () => {
9+
let service: BlobSinkServer;
10+
11+
beforeEach(async () => {
12+
service = new BlobSinkServer({
13+
port: 0, // Using port 0 lets the OS assign a random available port
14+
});
15+
await service.start();
16+
});
17+
18+
afterEach(async () => {
19+
await service.stop();
20+
});
21+
22+
describe('should store and retrieve a blob sidecar', () => {
23+
const testFields = [Fr.random(), Fr.random(), Fr.random()];
24+
const testFields2 = [Fr.random(), Fr.random(), Fr.random()];
25+
const blob = Blob.fromFields(testFields);
26+
const blob2 = Blob.fromFields(testFields2);
27+
const blockId = '0x1234';
28+
29+
beforeEach(async () => {
30+
// Post the blob
31+
const postResponse = await request(service.getApp())
32+
.post('/blob_sidecar')
33+
.send({
34+
// eslint-disable-next-line camelcase
35+
block_id: blockId,
36+
blobs: [
37+
{
38+
index: 0,
39+
blob: blob.toBuffer(),
40+
},
41+
{
42+
index: 1,
43+
blob: blob2.toBuffer(),
44+
},
45+
],
46+
});
47+
48+
expect(postResponse.status).toBe(200);
49+
});
50+
51+
it('should retrieve the blob', async () => {
52+
// Retrieve the blob
53+
const getResponse = await request(service.getApp()).get(`/eth/v1/beacon/blob_sidecars/${blockId}`);
54+
55+
expect(getResponse.status).toBe(200);
56+
57+
// Convert the response blob back to a Blob object and verify it matches
58+
const retrievedBlobs = getResponse.body.data;
59+
60+
const retrievedBlob = Blob.fromBuffer(Buffer.from(retrievedBlobs[0].blob, 'hex'));
61+
const retrievedBlob2 = Blob.fromBuffer(Buffer.from(retrievedBlobs[1].blob, 'hex'));
62+
expect(retrievedBlob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
63+
expect(retrievedBlob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
64+
expect(retrievedBlob2.fieldsHash.toString()).toBe(blob2.fieldsHash.toString());
65+
expect(retrievedBlob2.commitment.toString('hex')).toBe(blob2.commitment.toString('hex'));
66+
});
67+
68+
it('should retrieve specific indicies', async () => {
69+
// We can also request specific indicies
70+
const getWithIndicies = await request(service.getApp()).get(
71+
`/eth/v1/beacon/blob_sidecars/${blockId}?indices=0,1`,
72+
);
73+
74+
expect(getWithIndicies.status).toBe(200);
75+
expect(getWithIndicies.body.data.length).toBe(2);
76+
77+
const retrievedBlobs = getWithIndicies.body.data;
78+
const retrievedBlob = Blob.fromBuffer(Buffer.from(retrievedBlobs[0].blob, 'hex'));
79+
const retrievedBlob2 = Blob.fromBuffer(Buffer.from(retrievedBlobs[1].blob, 'hex'));
80+
expect(retrievedBlob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
81+
expect(retrievedBlob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
82+
expect(retrievedBlob2.fieldsHash.toString()).toBe(blob2.fieldsHash.toString());
83+
expect(retrievedBlob2.commitment.toString('hex')).toBe(blob2.commitment.toString('hex'));
84+
});
85+
86+
it('should retreive a single index', async () => {
87+
const getWithIndicies = await request(service.getApp()).get(`/eth/v1/beacon/blob_sidecars/${blockId}?indices=1`);
88+
89+
expect(getWithIndicies.status).toBe(200);
90+
expect(getWithIndicies.body.data.length).toBe(1);
91+
92+
const retrievedBlobs = getWithIndicies.body.data;
93+
const retrievedBlob = Blob.fromBuffer(Buffer.from(retrievedBlobs[0].blob, 'hex'));
94+
expect(retrievedBlob.fieldsHash.toString()).toBe(blob2.fieldsHash.toString());
95+
expect(retrievedBlob.commitment.toString('hex')).toBe(blob2.commitment.toString('hex'));
96+
});
97+
});
98+
99+
it('should return an error if invalid indicies are provided', async () => {
100+
const blockId = '0x1234';
101+
102+
const response = await request(service.getApp()).get(`/eth/v1/beacon/blob_sidecars/${blockId}?indices=word`);
103+
expect(response.status).toBe(400);
104+
expect(response.body.error).toBe('Invalid indices parameter');
105+
});
106+
107+
it('should return an error if the block ID is invalid (POST)', async () => {
108+
const response = await request(service.getApp()).post('/blob_sidecar').send({
109+
// eslint-disable-next-line camelcase
110+
block_id: undefined,
111+
});
112+
113+
expect(response.status).toBe(400);
114+
});
115+
116+
it('should return an error if the block ID is invalid (GET)', async () => {
117+
const response = await request(service.getApp()).get('/eth/v1/beacon/blob_sidecars/invalid-id');
118+
119+
expect(response.status).toBe(400);
120+
});
121+
122+
it('should return 404 for non-existent blob', async () => {
123+
const response = await request(service.getApp()).get('/eth/v1/beacon/blob_sidecars/0x999999');
124+
125+
expect(response.status).toBe(404);
126+
});
127+
128+
it('should reject negative block IDs', async () => {
129+
const response = await request(service.getApp()).get('/eth/v1/beacon/blob_sidecars/-123');
130+
131+
expect(response.status).toBe(400);
132+
expect(response.body.error).toBe('Invalid block_id parameter');
133+
});
134+
});

‎yarn-project/blob-sink/src/blobstore/blob_store_test_suite.ts

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import { Blob } from '@aztec/foundation/blob';
2+
import { Fr } from '@aztec/foundation/fields';
3+
4+
import { BlobWithIndex } from '../types/index.js';
5+
import { type BlobStore } from './interface.js';
6+
7+
export function describeBlobStore(getBlobStore: () => BlobStore) {
8+
let blobStore: BlobStore;
9+
10+
beforeEach(() => {
11+
blobStore = getBlobStore();
12+
});
13+
14+
it('should store and retrieve a blob', async () => {
15+
// Create a test blob with random fields
16+
const testFields = [Fr.random(), Fr.random(), Fr.random()];
17+
const blob = Blob.fromFields(testFields);
18+
const blockId = '0x12345';
19+
const blobWithIndex = new BlobWithIndex(blob, 0);
20+
21+
// Store the blob
22+
await blobStore.addBlobSidecars(blockId, [blobWithIndex]);
23+
24+
// Retrieve the blob
25+
const retrievedBlobs = await blobStore.getBlobSidecars(blockId);
26+
const [retrievedBlob] = retrievedBlobs!;
27+
28+
// Verify the blob was retrieved and matches
29+
expect(retrievedBlob).toBeDefined();
30+
expect(retrievedBlob.blob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
31+
expect(retrievedBlob.blob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
32+
});
33+
34+
it('Should allow requesting a specific index of blob', async () => {
35+
const testFields = [Fr.random(), Fr.random(), Fr.random()];
36+
const blob = Blob.fromFields(testFields);
37+
const blockId = '0x12345';
38+
const blobWithIndex = new BlobWithIndex(blob, 0);
39+
const blobWithIndex2 = new BlobWithIndex(blob, 1);
40+
41+
await blobStore.addBlobSidecars(blockId, [blobWithIndex, blobWithIndex2]);
42+
43+
const retrievedBlobs = await blobStore.getBlobSidecars(blockId, [0]);
44+
const [retrievedBlob] = retrievedBlobs!;
45+
46+
expect(retrievedBlob.blob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
47+
expect(retrievedBlob.blob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
48+
49+
const retrievedBlobs2 = await blobStore.getBlobSidecars(blockId, [1]);
50+
const [retrievedBlob2] = retrievedBlobs2!;
51+
52+
expect(retrievedBlob2.blob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
53+
expect(retrievedBlob2.blob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
54+
});
55+
56+
it('Differentiate between blockHash and slot', async () => {
57+
const testFields = [Fr.random(), Fr.random(), Fr.random()];
58+
const testFieldsSlot = [Fr.random(), Fr.random(), Fr.random()];
59+
const blob = Blob.fromFields(testFields);
60+
const blobSlot = Blob.fromFields(testFieldsSlot);
61+
const blockId = '0x12345';
62+
const slot = '12345';
63+
const blobWithIndex = new BlobWithIndex(blob, 0);
64+
const blobWithIndexSlot = new BlobWithIndex(blobSlot, 0);
65+
66+
await blobStore.addBlobSidecars(blockId, [blobWithIndex]);
67+
await blobStore.addBlobSidecars(slot, [blobWithIndexSlot]);
68+
69+
const retrievedBlobs = await blobStore.getBlobSidecars(blockId, [0]);
70+
const [retrievedBlob] = retrievedBlobs!;
71+
72+
expect(retrievedBlob.blob.fieldsHash.toString()).toBe(blob.fieldsHash.toString());
73+
expect(retrievedBlob.blob.commitment.toString('hex')).toBe(blob.commitment.toString('hex'));
74+
75+
const retrievedBlobs2 = await blobStore.getBlobSidecars(slot, [0]);
76+
const [retrievedBlob2] = retrievedBlobs2!;
77+
78+
expect(retrievedBlob2.blob.fieldsHash.toString()).toBe(blobSlot.fieldsHash.toString());
79+
expect(retrievedBlob2.blob.commitment.toString('hex')).toBe(blobSlot.commitment.toString('hex'));
80+
});
81+
82+
it('should return undefined for non-existent blob', async () => {
83+
const nonExistentBlob = await blobStore.getBlobSidecars('999999');
84+
expect(nonExistentBlob).toBeUndefined();
85+
});
86+
87+
it('should handle multiple blobs with different block IDs', async () => {
88+
// Create two different blobs
89+
const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
90+
const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
91+
const blobWithIndex1 = new BlobWithIndex(blob1, 0);
92+
const blobWithIndex2 = new BlobWithIndex(blob2, 0);
93+
94+
// Store both blobs
95+
await blobStore.addBlobSidecars('1', [blobWithIndex1]);
96+
await blobStore.addBlobSidecars('2', [blobWithIndex2]);
97+
98+
// Retrieve and verify both blobs
99+
const retrieved1 = await blobStore.getBlobSidecars('1');
100+
const retrieved2 = await blobStore.getBlobSidecars('2');
101+
const [retrievedBlob1] = retrieved1!;
102+
const [retrievedBlob2] = retrieved2!;
103+
104+
expect(retrievedBlob1.blob.commitment.toString('hex')).toBe(blob1.commitment.toString('hex'));
105+
expect(retrievedBlob2.blob.commitment.toString('hex')).toBe(blob2.commitment.toString('hex'));
106+
});
107+
108+
it('should overwrite blob when using same block ID', async () => {
109+
// Create two different blobs
110+
const originalBlob = Blob.fromFields([Fr.random()]);
111+
const newBlob = Blob.fromFields([Fr.random(), Fr.random()]);
112+
const blockId = '1';
113+
const originalBlobWithIndex = new BlobWithIndex(originalBlob, 0);
114+
const newBlobWithIndex = new BlobWithIndex(newBlob, 0);
115+
116+
// Store original blob
117+
await blobStore.addBlobSidecars(blockId, [originalBlobWithIndex]);
118+
119+
// Overwrite with new blob
120+
await blobStore.addBlobSidecars(blockId, [newBlobWithIndex]);
121+
122+
// Retrieve and verify it's the new blob
123+
const retrievedBlobs = await blobStore.getBlobSidecars(blockId);
124+
const [retrievedBlob] = retrievedBlobs!;
125+
expect(retrievedBlob.blob.commitment.toString('hex')).toBe(newBlob.commitment.toString('hex'));
126+
expect(retrievedBlob.blob.commitment.toString('hex')).not.toBe(originalBlob.commitment.toString('hex'));
127+
});
128+
129+
it('should handle multiple blobs with the same block ID', async () => {
130+
const blob1 = Blob.fromFields([Fr.random()]);
131+
const blob2 = Blob.fromFields([Fr.random()]);
132+
const blobWithIndex1 = new BlobWithIndex(blob1, 0);
133+
const blobWithIndex2 = new BlobWithIndex(blob2, 0);
134+
135+
await blobStore.addBlobSidecars('1', [blobWithIndex1, blobWithIndex2]);
136+
const retrievedBlobs = await blobStore.getBlobSidecars('1');
137+
const [retrievedBlob1, retrievedBlob2] = retrievedBlobs!;
138+
139+
expect(retrievedBlob1.blob.commitment.toString('hex')).toBe(blob1.commitment.toString('hex'));
140+
expect(retrievedBlob2.blob.commitment.toString('hex')).toBe(blob2.commitment.toString('hex'));
141+
});
142+
}

‎yarn-project/blob-sink/src/blobstore/disk_blob_store.test.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { openTmpStore } from '@aztec/kv-store/lmdb';
2+
3+
import { describeBlobStore } from './blob_store_test_suite.js';
4+
import { DiskBlobStore } from './disk_blob_store.js';
5+
6+
describe('DiskBlobStore', () => {
7+
describeBlobStore(() => new DiskBlobStore(openTmpStore()));
8+
});

0 commit comments

Comments
 (0)