feat: rename to data-manager, save car to -car instead of /car, integ…

…rate minio for local s3 testing

.env.example

@@ -1,9 +1,10 @@
 HOST="127.0.0.1"
 PORT="1234"
 JWT_SECRET="foo"
-DB_CONNECTION_STRING="postgresql://user:[email protected]:5432/data-uploader"
+DB_CONNECTION_STRING="postgresql://user:[email protected]:5432/data-manager"
 W3UP_PRINCIPAL_KEY="foo"
 W3UP_DELEGATION_PROOF="foo"
-S3_BUCKET="foo"
-S3_ACCESS_KEY_ID="foo"
-S3_SECRET_ACCESS_KEY="foo"
+S3_ENDPOINT="http://127.0.0.1:9000"
+S3_BUCKET="data-manager-test-bucket"
+S3_ACCESS_KEY_ID="data-manager-test-access-key"
+S3_SECRET_ACCESS_KEY="data-manager-test-secret-key"

.prettierrc

@@ -5,7 +5,7 @@
     "semi": true,
     "overrides": [
         {
-            "files": ["*.md"],
+            "files": ["*.md", "*.yaml", "*.yml"],
             "options": {
                 "printWidth": 80,
                 "proseWrap": "always",

Dockerfile

@@ -24,9 +24,9 @@ FROM base as runner
 COPY --from=prod-deps /app/node_modules /app/node_modules
 COPY --from=build /app/out/index.mjs /app/index.mjs
 
-RUN addgroup -S data-uploader-runners
-RUN adduser -S -G data-uploader-runners data-uploader-runner
-USER data-uploader-runner
+RUN addgroup -S data-manager-runners
+RUN adduser -S -G data-manager-runners data-manager-runner
+USER data-manager-runner
 
 ARG HOST
 ENV HOST=$HOST

README.md

@@ -17,7 +17,7 @@
     <img src="https://img.shields.io/badge/License-GPLv3-blue.svg" alt="License: GPL v3">
 </p>
 
-# Carrot data uploader
+# Carrot data manager
 
 This service is responsible for managing data in the Carrot protocol, which
 primarily falls into two categories at the time of writing:
@@ -67,13 +67,13 @@ Data in Carrot is mainly stored in two locations:
    API takes the raw input JSON, encodes it into the IPFS CAR format and
    determines the raw data CID. Both the raw content and the CAR file are
    uploaded to the S3 bucket using the CID as the base key (the raw content uses
-   the CID itself as the key, while the CAR is uploaded under `$CID/car`).
+   the CID itself as the key, while the CAR is uploaded under `$CID-car`).
 
 2. **`/data/ipfs`:** this endpoint persists limbo data and replicates it to
    IPFS/Filecoin. The API accepts a single parameter `cid` which must refer to
    some limbo data that the caller wants to persist to IPFS/Filecoin. The API
    fetches the CAR associated with the passed CID (stored on the S3 bucket under
-   `$CID/car`) and stores the fetched CAR file on IPFS/Filecoin through
+   `$CID-car`) and stores the fetched CAR file on IPFS/Filecoin through
    web3.storage's w3up service. The resulting upload CID is checked for
    consistency and if everything is fine the raw data is also persisted on the
    S3 bucket while the CAR is deleted from there.
@@ -119,7 +119,7 @@ template's code is referencing data the doesn't exist anywhere**.
 
 The best solution to avoid this scenario is to handle both limbo data addition
 and persistent data addition in the same place, and this place is the
-`data-uploader` service. Adding data to limbo will cause the `data-uploader`
+`data-manager` service. Adding data to limbo will cause the `data-manager`
 service to calculate this data's CID by creating an IPFS CAR containing the
 data, and returning this CID to the caller. **It's then responsibility of the
 caller to use that CID to reference the limbo data**. As long as the caller does

docker-compose.yaml

@@ -1,10 +1,20 @@
 version: "3"
 services:
-    postgres:
-        container_name: postgres
-        image: postgres:latest
-        ports:
-            - 127.0.0.1:5432:5432
-        environment:
-            POSTGRES_USER: user
-            POSTGRES_PASSWORD: password
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    ports:
+      - 127.0.0.1:5432:5432
+    environment:
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+  minio:
+    container_name: minio
+    image: bitnami/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      - MINIO_ROOT_USER=user
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DEFAULT_BUCKETS=data-manager-test-bucket

package.json

@@ -1,11 +1,11 @@
 {
     "type": "module",
     "private": true,
-    "name": "@carrot-kpi/data-uploader",
+    "name": "@carrot-kpi/data-manager",
     "version": "0.1.0",
     "author": "Federico Luzzi <[email protected]>",
     "license": "GPL-3.0-or-later",
-    "description": "A service to facilitate generic data uploads to different services.",
+    "description": "A service to facilitate data management in Carrot.",
     "scripts": {
         "prepare": "node ./scripts/prepare.js",
         "commitlint": "commitlint -e",

src/constants.ts

@@ -1,4 +1,4 @@
-export const JWT_ISSUER = "carrot-data-uploader";
+export const JWT_ISSUER = "carrot-data-manager";
 export const NONCE_LENGTH_BYTES = 32;
 export const SCOPE_S3 = "s3"
 export const SCOPE_IPFS = "ipfs"

src/index.ts

@@ -54,9 +54,9 @@ const start = async () => {
             plugin: HapiSwaggerPlugin,
             options: {
                 info: {
-                    title: "Data uploader API",
+                    title: "Data manager API",
                     version: "1.0.0",
-                    description: "An API to access various storage services.",
+                    description: "An API to manage data in Carrot.",
                     contact: {
                         name: "Carrot Labs",
                         email: "[email protected]",
@@ -79,10 +79,12 @@ const start = async () => {
         delegationProof: W3UP_DELEGATION_PROOF,
     });
 
+    const S3_ENDPOINT = process.env.S3_ENDPOINT;
     const S3_BUCKET = requireEnv({ name: "S3_BUCKET" });
     const S3_ACCESS_KEY_ID = requireEnv({ name: "S3_ACCESS_KEY_ID" });
     const S3_SECRET_ACCESS_KEY = requireEnv({ name: "S3_SECRET_ACCESS_KEY" });
     const s3Client = getS3Client({
+        endpoint: S3_ENDPOINT,
         accessKeyId: S3_ACCESS_KEY_ID,
         secretAccessKey: S3_SECRET_ACCESS_KEY,
     });

src/routes/data/ipfs.ts

@@ -102,12 +102,12 @@ export const getIPFSDataRoute = async ({
             try {
                 const getCAR = new GetObjectCommand({
                     Bucket: s3Bucket,
-                    Key: `${cid}/car`,
+                    Key: `${cid}-car`,
                 });
                 const output = await s3Client.send(getCAR);
                 if (!output.Body)
                     throw new Error(
-                        `Could not fetch object with key "${cid}/car" from S3`,
+                        `Could not fetch object with key "${cid}-car" from S3`,
                     );
                 car = output.Body;
             } catch (error) {
@@ -185,13 +185,13 @@ export const getIPFSDataRoute = async ({
             try {
                 const deleteCAR = new DeleteObjectCommand({
                     Bucket: s3Bucket,
-                    Key: `${cid}/car`,
+                    Key: `${cid}-car`,
                 });
                 await s3Client.send(deleteCAR);
             } catch (error) {
                 request.logger.error(
                     error,
-                    `Could not delete CAR with key "${cid}/car" from S3`,
+                    `Could not delete CAR with key "${cid}-car" from S3`,
                 );
                 return badGateway("Could not upload data to IPFS");
             }

src/routes/data/s3.ts

@@ -82,7 +82,7 @@ export const getS3DataRoute = async ({
             try {
                 const put = new PutObjectCommand({
                     Bucket: s3Bucket,
-                    Key: `${cid}/car`,
+                    Key: `${cid}-car`,
                     Body: Readable.fromWeb(car.stream()),
                     ContentLength: car.size,
                     ContentType: "application/vnd.ipld.car",

src/utils.ts

@@ -62,16 +62,19 @@ export const getW3UpClient = async ({
 };
 
 interface GetS3ClientParams {
+    endpoint?: string;
     accessKeyId: string;
     secretAccessKey: string;
 }
 
 export const getS3Client = ({
+    endpoint,
     accessKeyId,
     secretAccessKey,
 }: GetS3ClientParams): S3Client => {
     return new S3Client({
         region: "us-east-1",
+        endpoint,
         credentials: {
             accessKeyId,
             secretAccessKey,
@@ -205,7 +208,7 @@ interface GetAuthenticationSchemeParams {
     jwtSecretKey: string;
 }
 
-interface DataUploaderJWTPayload extends JwtPayload {
+interface DataManagerJWTPayload extends JwtPayload {
     scp?: string[];
 }
 
@@ -232,13 +235,13 @@ export const getAuthenticationScheme = ({
 
             try {
                 const jwt = authorization.split(" ")[1];
-                const payload: DataUploaderJWTPayload = jsonwebtoken.verify(
+                const payload: DataManagerJWTPayload = jsonwebtoken.verify(
                     jwt,
                     jwtSecretKey,
                     {
                         issuer: JWT_ISSUER,
                     },
-                ) as DataUploaderJWTPayload;
+                ) as DataManagerJWTPayload;
 
                 return h.authenticated({
                     credentials: {