Add backbeat route apis

.github/authdata.json

@@ -0,0 +1,21 @@
+{
+    "accounts": [{
+        "name": "bart",
+        "arn": "aws::iam:123456789012:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+        "displayName": "bart",
+        "keys": {
+            "access": "accessKey1",
+            "secret": "verySecretKey1"
+        }
+    }, {
+        "name": "lisa",
+        "arn": "aws::iam:123456789013:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
+        "displayName": "lisa",
+        "keys": {
+            "access": "accessKey2",
+            "secret": "verySecretKey2"
+        }
+    }]
+}

.github/docker-compose.backbeat.yml

@@ -0,0 +1,216 @@
+version: '3.8'
+
+networks:
+  backbeat-net:
+    driver: bridge
+
+services:
+  zookeeper:
+    image: zookeeper:3.8
+    platform: linux/amd64
+    ports:
+      - "2181:2181"
+    networks:
+      - backbeat-net
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+
+  kafka:
+    image: confluentinc/cp-kafka:7.4.0
+    platform: linux/amd64
+    ports:
+      - "9092:9092"
+    networks:
+      - backbeat-net
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    depends_on:
+      - zookeeper
+
+  mongodb:
+    image: mongo:4.4
+    platform: linux/amd64
+    ports:
+      - "27018:27018"
+    networks:
+      - backbeat-net
+    command: mongod --port 27018 --replSet rs0 --bind_ip_all
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+
+  mongodb-init:
+    image: mongo:4.4
+    platform: linux/amd64
+    networks:
+      - backbeat-net
+    depends_on:
+      mongodb:
+        condition: service_healthy
+    command: >
+      mongo --host mongodb:27018 --eval
+      'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})'
+    restart: "no"
+
+  redis:
+    image: redis:7-alpine
+    platform: linux/amd64
+    ports:
+      - "6379:6379"
+    networks:
+      - backbeat-net
+
+  cloudserver:
+    image: ghcr.io/scality/cloudserver:9.1.4
+    platform: linux/amd64
+    ports:
+      - "8000:8000"
+    networks:
+      - backbeat-net
+    environment:
+      - S3VAULT=mem
+      - S3METADATA=mongodb
+      - S3DATA=mem
+      - MONGODB_HOSTS=mongodb:27018
+      - MONGODB_RS=rs0
+      - REMOTE_MANAGEMENT_DISABLE=true
+      - LOG_LEVEL=info
+      - CRR_METRICS_HOST=backbeat-api
+      - CRR_METRICS_PORT=8900
+    depends_on:
+      mongodb-init:
+        condition: service_completed_successfully
+      backbeat-api:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+
+  backbeat-api:
+    image: ghcr.io/scality/backbeat:9.1.3
+    platform: linux/amd64
+    ports:
+      - "8900:8900"
+    networks:
+      - backbeat-net
+    environment:
+      # Kafka configuration
+      KAFKA_HOSTS: kafka:9092
+      KAFKA_COMPRESSION_TYPE: snappy
+
+      # Zookeeper configuration
+      ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat
+      ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true"
+
+      # Redis configuration
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+
+      # MongoDB configuration
+      MONGODB_HOSTS: mongodb:27018
+      MONGODB_DATABASE: metadata
+
+      # Management backend
+      MANAGEMENT_BACKEND: operator
+      REMOTE_MANAGEMENT_DISABLE: "true"
+
+      # Cloudserver configuration
+      CLOUDSERVER_HOST: cloudserver
+      CLOUDSERVER_PORT: 8000
+
+      # Replication source S3
+      EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver
+      EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000
+      EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service
+      EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication
+
+      # Replication destination
+      EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service
+      EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication
+      EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001
+      EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}'
+
+      # Lifecycle
+      EXTENSIONS_LIFECYCLE_AUTH_TYPE: service
+      EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: service-lifecycle
+
+      # Healthchecks
+      HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0"
+
+      # Logging
+      LOG_LEVEL: info
+    depends_on:
+      mongodb-init:
+        condition: service_completed_successfully
+      kafka:
+        condition: service_started
+      redis:
+        condition: service_started
+    command: node bin/backbeat.js
+
+  backbeat-queue-processor:
+    image: ghcr.io/scality/backbeat:9.1.3
+    platform: linux/amd64
+    ports:
+      - "4043:4043"
+    networks:
+      - backbeat-net
+    environment:
+      # Kafka configuration
+      KAFKA_HOSTS: kafka:9092
+      KAFKA_COMPRESSION_TYPE: snappy
+
+      # Zookeeper configuration
+      ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat
+      ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true"
+
+      # Redis configuration
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+
+      # MongoDB configuration
+      MONGODB_HOSTS: mongodb:27018
+      MONGODB_DATABASE: metadata
+
+      # Management backend
+      MANAGEMENT_BACKEND: operator
+      REMOTE_MANAGEMENT_DISABLE: "true"
+
+      # Cloudserver configuration
+      CLOUDSERVER_HOST: cloudserver
+      CLOUDSERVER_PORT: 8000
+
+      # Replication source S3
+      EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver
+      EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000
+      EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service
+      EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication
+
+      # Replication destination
+      EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service
+      EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication
+      EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001
+      EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}'
+
+      # Probe server port
+      LIVENESS_PROBE_PORT: 4043
+
+      # Logging
+      LOG_LEVEL: info
+    depends_on:
+      mongodb-init:
+        condition: service_completed_successfully
+      cloudserver:
+        condition: service_started
+      backbeat-api:
+        condition: service_started
+    command: node extensions/replication/queueProcessor/task.js

.github/workflows/test.yml

@@ -128,3 +128,54 @@ jobs:
       - name: Stop Cloudserver
         if: always()
         run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down
+
+  test-backbeat-apis:
+    name: Test backbeat apis
+    runs-on: ubuntu-latest
+    needs: lint
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+
+      - name: Setup Smithy CLI
+        uses: ./.github/actions/setup-smithy
+
+      - name: Build
+        run: yarn build
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Start Backbeat with CloudServer
+        run: docker compose -f .github/docker-compose.backbeat.yml up -d
+
+      - name: Wait for Cloudserver to be ready
+        run: |
+          set -o pipefail
+          bash .github/scripts/wait_for_local_port.bash 8000 40
+
+      # - name: Wait for Backbeat API to be ready
+      #   run: |
+      #     set -o pipefail
+      #     bash .github/scripts/wait_for_local_port.bash 8900 60
+
+      - name: Run backbeat apis tests
+        run: yarn test:backbeat-apis
+
+      - name: Stop Backbeat and Cloudserver
+        if: always()
+        run: docker compose -f .github/docker-compose.backbeat.yml down

models/backbeatApis/checkConnection.smithy

@@ -0,0 +1,14 @@
+$version: "2.0"
+namespace cloudserver.client
+
+/// Health check endpoint for Backbeat API
+@readonly
+@http(method: "GET", uri: "/_/backbeat/api/healthcheck")
+operation CheckConnection {
+    input: CheckConnectionInput,
+    output: CheckConnectionOutput
+}
+
+structure CheckConnectionInput {}
+
+structure CheckConnectionOutput {}

models/backbeatApis/commonStructures.smithy

@@ -0,0 +1,29 @@
+$version: "2.0"
+namespace cloudserver.client
+
+/// Map containing status information for each location
+map LocationStatusMap {
+    key: String,
+    value: Document
+}
+
+/// Information about a failed replication object
+structure FailedObjectVersion {
+    /// Bucket name
+    Bucket: String,
+
+    /// Object key
+    Key: String,
+
+    /// Version ID
+    VersionId: String,
+
+    /// Storage class
+    StorageClass: String,
+
+    /// Object size in bytes
+    Size: Integer,
+
+    /// Last modified timestamp
+    LastModified: Timestamp
+}

models/backbeatApis/getFailedObject.smithy

@@ -0,0 +1,35 @@
+$version: "2.0"
+namespace cloudserver.client
+
+/// Get details of a specific failed Cross-Region Replication (CRR) object
+@readonly
+@http(method: "GET", uri: "/_/backbeat/api/crr/failed/{Bucket}/{Key+}")
+operation GetFailedObject {
+    input: GetFailedObjectInput,
+    output: GetFailedObjectOutput
+}
+
+structure GetFailedObjectInput {
+    /// Bucket name
+    @required
+    @httpLabel
+    Bucket: String,
+
+    /// Object key
+    @required
+    @httpLabel
+    Key: String,
+
+    /// Version ID of the object
+    @required
+    @httpQuery("versionId")
+    VersionId: String
+}
+
+structure GetFailedObjectOutput {
+    /// Whether the list is truncated
+    IsTruncated: Boolean,
+
+    /// List of failed object versions
+    Versions: FailedObjectVersions
+}

models/backbeatApis/getLocationsIngestionStatus.smithy

@@ -0,0 +1,17 @@
+$version: "2.0"
+namespace cloudserver.client
+
+/// Get the status of all ingestion locations
+@readonly
+@http(method: "GET", uri: "/_/backbeat/api/ingestion/status")
+operation GetLocationsIngestionStatus {
+    input: GetLocationsIngestionStatusInput,
+    output: GetLocationsIngestionStatusOutput
+}
+
+structure GetLocationsIngestionStatusInput {}
+
+structure GetLocationsIngestionStatusOutput {
+    /// Map of location names to their ingestion status information
+    status: LocationStatusMap
+}