feat: store car alongside content on s3 to then move it in a determin…

…istic way to ipfs

package.json

@@ -48,6 +48,7 @@
         "hapi-swagger": "^17.2.0",
         "joi": "^17.11.1",
         "jsonwebtoken": "^9.0.2",
+        "multiformats": "^13.0.1",
         "pg": "^8.11.3",
         "viem": "^2.2.0"
     }

src/constants.ts

@@ -1,15 +1,4 @@
 export const JWT_ISSUER = "carrot-data-uploader";
 export const NONCE_LENGTH_BYTES = 32;
-
-export enum DataFormat {
-    Json = "json",
-}
-
-export const MIME_TYPE: Record<DataFormat, string> = {
-    [DataFormat.Json]: "application/json",
-};
-
-export enum StorageService {
-    Ipfs = "ipfs",
-    S3 = "s3",
-}
+export const SCOPE_S3 = "s3"
+export const SCOPE_IPFS = "ipfs"

src/index.ts

@@ -16,7 +16,8 @@ import HapiVisionPlugin from "@hapi/vision";
 import HapiSwaggerPlugin from "hapi-swagger";
 import { getLoginMessageRoute } from "./routes/login-message";
 import { getTokenRoute } from "./routes/token";
-import { getDataRoutes } from "./routes/data";
+import { getS3DataRoute } from "./routes/data/s3.js";
+import { getIPFSDataRoute } from "./routes/data/ipfs.js";
 
 const DEV = process.env.NODE_ENV !== "production";
 
@@ -55,8 +56,7 @@ const start = async () => {
                 info: {
                     title: "Data uploader API",
                     version: "1.0.0",
-                    description:
-                        "An API to access various storage services.",
+                    description: "An API to access various storage services.",
                     contact: {
                         name: "Carrot Labs",
                         email: "[email protected]",
@@ -100,7 +100,13 @@ const start = async () => {
     server.route(getLoginMessageRoute({ dbClient }));
     server.route(getTokenRoute({ dbClient, jwtSecretKey: JWT_SECRET }));
     server.route(
-        await getDataRoutes({
+        await getS3DataRoute({
+            s3Client,
+            s3Bucket: S3_BUCKET,
+        }),
+    );
+    server.route(
+        await getIPFSDataRoute({
             w3UpClient,
             s3Client,
             s3Bucket: S3_BUCKET,

src/routes/data/ipfs.ts

@@ -0,0 +1,113 @@
+import { badGateway } from "@hapi/boom";
+import joi from "joi";
+import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import type { ServerRoute } from "@hapi/hapi";
+import type { Client as W3UpClient } from "@web3-storage/w3up-client";
+import { SCOPE_IPFS } from "../../constants";
+
+interface GetDataRoutesParams {
+    w3UpClient: W3UpClient;
+    s3Client: S3Client;
+    s3Bucket: string;
+}
+
+export const getIPFSDataRoute = async ({
+    w3UpClient,
+    s3Client,
+    s3Bucket,
+}: GetDataRoutesParams): Promise<ServerRoute> => {
+    return {
+        method: "POST",
+        path: "/data/ipfs",
+        options: {
+            plugins: {
+                "hapi-swagger": {
+                    responses: {
+                        401: {
+                            description: "Unauthorized caller.",
+                        },
+                        400: {
+                            description: "The request was not valid.",
+                        },
+                        502: {
+                            description: "The data could not be stored.",
+                        },
+                        200: {
+                            description: "The data was successfully stored.",
+                            schema: joi
+                                .object({
+                                    cid: joi
+                                        .string()
+                                        .label("The cid of the stored data."),
+                                })
+                                .required(),
+                        },
+                    },
+                },
+            },
+            description: "Stores JSON data on S3.",
+            notes: "Stores JSON data on S3. The data has to have previously been stored on S3 in a format that the service can understand and the caller must be explicitly authorized to call this endpoint.",
+            tags: ["api"],
+            auth: {
+                access: {
+                    scope: SCOPE_IPFS,
+                },
+            },
+            validate: {
+                headers: joi
+                    .object({
+                        authorization: joi
+                            .string()
+                            .required()
+                            .regex(
+                                /^Bearer [0-9a-zA-Z]*\.[0-9a-zA-Z]*\.[0-9a-zA-Z-_]*$/,
+                            ),
+                    })
+                    .unknown(),
+                payload: joi.object({
+                    cid: joi
+                        .string()
+                        .required()
+                        .description(
+                            "The CID of the object previously stored on S3 that needs to be persisted to IPFS.",
+                        ),
+                }),
+            },
+        },
+        handler: async (request, h) => {
+            const { cid } = request.payload as { cid: string };
+
+            let carStream: ReadableStream;
+            try {
+                const get = new GetObjectCommand({
+                    Bucket: s3Bucket,
+                    Key: `${cid}/car`,
+                });
+                const object = await s3Client.send(get);
+                if (!object.Body)
+                    throw new Error(
+                        `Could not fetch object with cid "${cid}/car" from S3`,
+                    );
+                carStream = object.Body.transformToWebStream();
+            } catch (error) {
+                request.logger.error(error, "Could not fetch CAR from S3");
+                return badGateway("Could not fetch CAR from S3");
+            }
+
+            try {
+                const cidFromUpload = await w3UpClient.uploadCAR({
+                    stream: () => carStream,
+                });
+                if (cidFromUpload.toV1().toString() !== cid)
+                    throw new Error(
+                        `CID mismatch: got ${cidFromUpload.toV1().toString()}, expected ${cid}`,
+                    );
+            } catch (error) {
+                request.logger.error(error, "Could not upload CAR to IPFS");
+                return badGateway("Could not upload data to IPFS");
+            }
+
+            return h.response({ cid }).code(200).type("application/json");
+        },
+    };
+};