diff --git a/.github/authdata.json b/.github/authdata.json new file mode 100644 index 00000000..36b2bf85 --- /dev/null +++ b/.github/authdata.json @@ -0,0 +1,21 @@ +{ + "accounts": [{ + "name": "bart", + "arn": "aws::iam:123456789012:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "displayName": "bart", + "keys": { + "access": "accessKey1", + "secret": "verySecretKey1" + } + }, { + "name": "lisa", + "arn": "aws::iam:123456789013:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", + "displayName": "lisa", + "keys": { + "access": "accessKey2", + "secret": "verySecretKey2" + } + }] +} diff --git a/.github/docker-compose.backbeat.yml b/.github/docker-compose.backbeat.yml new file mode 100644 index 00000000..fafad309 --- /dev/null +++ b/.github/docker-compose.backbeat.yml @@ -0,0 +1,216 @@ +version: '3.8' + +networks: + backbeat-net: + driver: bridge + +services: + zookeeper: + image: zookeeper:3.8 + platform: linux/amd64 + ports: + - "2181:2181" + networks: + - backbeat-net + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: confluentinc/cp-kafka:7.4.0 + platform: linux/amd64 + ports: + - "9092:9092" + networks: + - backbeat-net + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + depends_on: + - zookeeper + + mongodb: + image: mongo:4.4 + platform: linux/amd64 + ports: + - "27018:27018" + networks: + - backbeat-net + command: mongod --port 27018 --replSet rs0 --bind_ip_all + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + mongodb-init: + image: mongo:4.4 + platform: linux/amd64 + networks: + - backbeat-net + depends_on: + mongodb: + condition: service_healthy + command: > + mongo --host mongodb:27018 --eval + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' + restart: "no" + + redis: + image: redis:7-alpine + platform: linux/amd64 + ports: + - "6379:6379" + networks: + - backbeat-net + + cloudserver: + image: ghcr.io/scality/cloudserver:9.1.4 + platform: linux/amd64 + ports: + - "8000:8000" + networks: + - backbeat-net + environment: + - S3VAULT=mem + - S3METADATA=mongodb + - S3DATA=mem + - MONGODB_HOSTS=mongodb:27018 + - MONGODB_RS=rs0 + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + - CRR_METRICS_HOST=backbeat-api + - CRR_METRICS_PORT=8900 + depends_on: + mongodb-init: + condition: service_completed_successfully + backbeat-api: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/"] + interval: 5s + timeout: 5s + retries: 12 + + backbeat-api: + image: ghcr.io/scality/backbeat:9.1.3 + platform: linux/amd64 + ports: + - "8900:8900" + networks: + - backbeat-net + environment: + # Kafka configuration + KAFKA_HOSTS: kafka:9092 + KAFKA_COMPRESSION_TYPE: snappy + + # Zookeeper configuration + ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat + ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" + + # Redis configuration + REDIS_HOST: redis + REDIS_PORT: 6379 + + # MongoDB configuration + MONGODB_HOSTS: mongodb:27018 + MONGODB_DATABASE: metadata + + # Management backend + MANAGEMENT_BACKEND: operator + REMOTE_MANAGEMENT_DISABLE: "true" + + # Cloudserver configuration + CLOUDSERVER_HOST: cloudserver + CLOUDSERVER_PORT: 8000 + + # Replication source S3 + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication + + # Replication destination + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' + + # Lifecycle + EXTENSIONS_LIFECYCLE_AUTH_TYPE: service + EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: service-lifecycle + + # Healthchecks + HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" + + # Logging + LOG_LEVEL: info + depends_on: + mongodb-init: + condition: service_completed_successfully + kafka: + condition: service_started + redis: + condition: service_started + command: node bin/backbeat.js + + backbeat-queue-processor: + image: ghcr.io/scality/backbeat:9.1.3 + platform: linux/amd64 + ports: + - "4043:4043" + networks: + - backbeat-net + environment: + # Kafka configuration + KAFKA_HOSTS: kafka:9092 + KAFKA_COMPRESSION_TYPE: snappy + + # Zookeeper configuration + ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat + ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" + + # Redis configuration + REDIS_HOST: redis + REDIS_PORT: 6379 + + # MongoDB configuration + MONGODB_HOSTS: mongodb:27018 + MONGODB_DATABASE: metadata + + # Management backend + MANAGEMENT_BACKEND: operator + REMOTE_MANAGEMENT_DISABLE: "true" + + # Cloudserver configuration + CLOUDSERVER_HOST: cloudserver + CLOUDSERVER_PORT: 8000 + + # Replication source S3 + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication + + # Replication destination + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' + + # Probe server port + LIVENESS_PROBE_PORT: 4043 + + # Logging + LOG_LEVEL: info + depends_on: + mongodb-init: + condition: service_completed_successfully + cloudserver: + condition: service_started + backbeat-api: + condition: service_started + command: node extensions/replication/queueProcessor/task.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47e9a25b..359348d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -128,3 +128,54 @@ jobs: - name: Stop Cloudserver if: always() run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down + + test-backbeat-apis: + name: Test backbeat apis + runs-on: ubuntu-latest + needs: lint + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Setup Smithy CLI + uses: ./.github/actions/setup-smithy + + - name: Build + run: yarn build + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Start Backbeat with CloudServer + run: docker compose -f .github/docker-compose.backbeat.yml up -d + + - name: Wait for Cloudserver to be ready + run: | + set -o pipefail + bash .github/scripts/wait_for_local_port.bash 8000 40 + + # - name: Wait for Backbeat API to be ready + # run: | + # set -o pipefail + # bash .github/scripts/wait_for_local_port.bash 8900 60 + + - name: Run backbeat apis tests + run: yarn test:backbeat-apis + + - name: Stop Backbeat and Cloudserver + if: always() + run: docker compose -f .github/docker-compose.backbeat.yml down \ No newline at end of file diff --git a/models/backbeatApis/checkConnection.smithy b/models/backbeatApis/checkConnection.smithy new file mode 100644 index 00000000..098348bb --- /dev/null +++ b/models/backbeatApis/checkConnection.smithy @@ -0,0 +1,14 @@ +$version: "2.0" +namespace cloudserver.client + +/// Health check endpoint for Backbeat API +@readonly +@http(method: "GET", uri: "/_/backbeat/api/healthcheck") +operation CheckConnection { + input: CheckConnectionInput, + output: CheckConnectionOutput +} + +structure CheckConnectionInput {} + +structure CheckConnectionOutput {} diff --git a/models/backbeatApis/commonStructures.smithy b/models/backbeatApis/commonStructures.smithy new file mode 100644 index 00000000..3938c9b4 --- /dev/null +++ b/models/backbeatApis/commonStructures.smithy @@ -0,0 +1,29 @@ +$version: "2.0" +namespace cloudserver.client + +/// Map containing status information for each location +map LocationStatusMap { + key: String, + value: Document +} + +/// Information about a failed replication object +structure FailedObjectVersion { + /// Bucket name + Bucket: String, + + /// Object key + Key: String, + + /// Version ID + VersionId: String, + + /// Storage class + StorageClass: String, + + /// Object size in bytes + Size: Integer, + + /// Last modified timestamp + LastModified: Timestamp +} \ No newline at end of file diff --git a/models/backbeatApis/getFailedObject.smithy b/models/backbeatApis/getFailedObject.smithy new file mode 100644 index 00000000..b70fe714 --- /dev/null +++ b/models/backbeatApis/getFailedObject.smithy @@ -0,0 +1,35 @@ +$version: "2.0" +namespace cloudserver.client + +/// Get details of a specific failed Cross-Region Replication (CRR) object +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed/{Bucket}/{Key+}") +operation GetFailedObject { + input: GetFailedObjectInput, + output: GetFailedObjectOutput +} + +structure GetFailedObjectInput { + /// Bucket name + @required + @httpLabel + Bucket: String, + + /// Object key + @required + @httpLabel + Key: String, + + /// Version ID of the object + @required + @httpQuery("versionId") + VersionId: String +} + +structure GetFailedObjectOutput { + /// Whether the list is truncated + IsTruncated: Boolean, + + /// List of failed object versions + Versions: FailedObjectVersions +} diff --git a/models/backbeatApis/getLocationsIngestionStatus.smithy b/models/backbeatApis/getLocationsIngestionStatus.smithy new file mode 100644 index 00000000..ed0a60e1 --- /dev/null +++ b/models/backbeatApis/getLocationsIngestionStatus.smithy @@ -0,0 +1,17 @@ +$version: "2.0" +namespace cloudserver.client + +/// Get the status of all ingestion locations +@readonly +@http(method: "GET", uri: "/_/backbeat/api/ingestion/status") +operation GetLocationsIngestionStatus { + input: GetLocationsIngestionStatusInput, + output: GetLocationsIngestionStatusOutput +} + +structure GetLocationsIngestionStatusInput {} + +structure GetLocationsIngestionStatusOutput { + /// Map of location names to their ingestion status information + status: LocationStatusMap +} diff --git a/models/backbeatApis/getLocationsStatus.smithy b/models/backbeatApis/getLocationsStatus.smithy new file mode 100644 index 00000000..1fec3a94 --- /dev/null +++ b/models/backbeatApis/getLocationsStatus.smithy @@ -0,0 +1,17 @@ +$version: "2.0" +namespace cloudserver.client + +/// Get the status of all replication locations +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/status") +operation GetLocationsStatus { + input: GetLocationsStatusInput, + output: GetLocationsStatusOutput +} + +structure GetLocationsStatusInput {} + +structure GetLocationsStatusOutput { + /// Map of location names to their status information + status: LocationStatusMap +} diff --git a/models/backbeatApis/listFailed.smithy b/models/backbeatApis/listFailed.smithy new file mode 100644 index 00000000..98e08495 --- /dev/null +++ b/models/backbeatApis/listFailed.smithy @@ -0,0 +1,35 @@ +$version: "2.0" +namespace cloudserver.client + +/// List failed Cross-Region Replication (CRR) objects +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed") +operation ListFailed { + input: ListFailedInput, + output: ListFailedOutput +} + +structure ListFailedInput { + /// Marker for pagination + @httpQuery("marker") + Marker: String, + + /// Site name to filter by + @httpQuery("sitename") + Sitename: String +} + +structure ListFailedOutput { + /// Whether the list is truncated + IsTruncated: Boolean, + + /// Marker for the next page + NextMarker: String, + + /// List of failed object versions + Versions: FailedObjectVersions +} + +list FailedObjectVersions { + member: FailedObjectVersion +} diff --git a/models/backbeatApis/pauseAllIngestionSites.smithy b/models/backbeatApis/pauseAllIngestionSites.smithy new file mode 100644 index 00000000..8f876061 --- /dev/null +++ b/models/backbeatApis/pauseAllIngestionSites.smithy @@ -0,0 +1,20 @@ +$version: "2.0" +namespace cloudserver.client + +/// Pause ingestion for all ingestion sites +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause") +operation PauseAllIngestionSites { + input: PauseAllIngestionSitesInput, + output: PauseAllIngestionSitesOutput +} + +structure PauseAllIngestionSitesInput { + /// Optional request body + @httpPayload + Body: Blob +} + +structure PauseAllIngestionSitesOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/pauseAllSites.smithy b/models/backbeatApis/pauseAllSites.smithy new file mode 100644 index 00000000..1d1c7409 --- /dev/null +++ b/models/backbeatApis/pauseAllSites.smithy @@ -0,0 +1,20 @@ +$version: "2.0" +namespace cloudserver.client + +/// Pause replication for all Cross-Region Replication (CRR) sites +@http(method: "POST", uri: "/_/backbeat/api/crr/pause") +operation PauseAllSites { + input: PauseAllSitesInput, + output: PauseAllSitesOutput +} + +structure PauseAllSitesInput { + /// Optional request body + @httpPayload + Body: Blob +} + +structure PauseAllSitesOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/pauseIngestionSite.smithy b/models/backbeatApis/pauseIngestionSite.smithy new file mode 100644 index 00000000..b2844122 --- /dev/null +++ b/models/backbeatApis/pauseIngestionSite.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Pause ingestion for a specific ingestion site +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause/{Site}") +operation PauseIngestionSite { + input: PauseIngestionSiteInput, + output: PauseIngestionSiteOutput +} + +structure PauseIngestionSiteInput { + /// Site name to pause + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure PauseIngestionSiteOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/pauseSite.smithy b/models/backbeatApis/pauseSite.smithy new file mode 100644 index 00000000..7273fbbb --- /dev/null +++ b/models/backbeatApis/pauseSite.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Pause replication for a specific Cross-Region Replication (CRR) site +@http(method: "POST", uri: "/_/backbeat/api/crr/pause/{Site}") +operation PauseSite { + input: PauseSiteInput, + output: PauseSiteOutput +} + +structure PauseSiteInput { + /// Site name to pause + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure PauseSiteOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/resumeAllIngestionSites.smithy b/models/backbeatApis/resumeAllIngestionSites.smithy new file mode 100644 index 00000000..9b10f8c3 --- /dev/null +++ b/models/backbeatApis/resumeAllIngestionSites.smithy @@ -0,0 +1,20 @@ +$version: "2.0" +namespace cloudserver.client + +/// Resume ingestion for all ingestion sites +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume") +operation ResumeAllIngestionSites { + input: ResumeAllIngestionSitesInput, + output: ResumeAllIngestionSitesOutput +} + +structure ResumeAllIngestionSitesInput { + /// Optional request body + @httpPayload + Body: Blob +} + +structure ResumeAllIngestionSitesOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/resumeAllSites.smithy b/models/backbeatApis/resumeAllSites.smithy new file mode 100644 index 00000000..247d85c2 --- /dev/null +++ b/models/backbeatApis/resumeAllSites.smithy @@ -0,0 +1,20 @@ +$version: "2.0" +namespace cloudserver.client + +/// Resume replication for all Cross-Region Replication (CRR) sites +@http(method: "POST", uri: "/_/backbeat/api/crr/resume") +operation ResumeAllSites { + input: ResumeAllSitesInput, + output: ResumeAllSitesOutput +} + +structure ResumeAllSitesInput { + /// Optional request body + @httpPayload + Body: Blob +} + +structure ResumeAllSitesOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/resumeIngestionSite.smithy b/models/backbeatApis/resumeIngestionSite.smithy new file mode 100644 index 00000000..c2dae834 --- /dev/null +++ b/models/backbeatApis/resumeIngestionSite.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Resume ingestion for a specific ingestion site +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}") +operation ResumeIngestionSite { + input: ResumeIngestionSiteInput, + output: ResumeIngestionSiteOutput +} + +structure ResumeIngestionSiteInput { + /// Site name to resume + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure ResumeIngestionSiteOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/resumeSite.smithy b/models/backbeatApis/resumeSite.smithy new file mode 100644 index 00000000..c144473a --- /dev/null +++ b/models/backbeatApis/resumeSite.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Resume replication for a specific Cross-Region Replication (CRR) site +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}") +operation ResumeSite { + input: ResumeSiteInput, + output: ResumeSiteOutput +} + +structure ResumeSiteInput { + /// Site name to resume + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure ResumeSiteOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/retryFailedObjects.smithy b/models/backbeatApis/retryFailedObjects.smithy new file mode 100644 index 00000000..c3a83a98 --- /dev/null +++ b/models/backbeatApis/retryFailedObjects.smithy @@ -0,0 +1,23 @@ +$version: "2.0" +namespace cloudserver.client + +/// Retry failed Cross-Region Replication (CRR) objects +@http(method: "POST", uri: "/_/backbeat/api/crr/failed") +operation RetryFailedObjects { + input: RetryFailedObjectsInput, + output: RetryFailedObjectsOutput +} + +structure RetryFailedObjectsInput { + /// Request body containing the list of failed objects to retry + @required + @httpPayload + Body: Blob +} + +structure RetryFailedObjectsOutput { + /// List of retry results directly in response body as an array. + /// Expected structure: Array<{Bucket, Key, VersionId, StorageClass, Size, LastModified, ReplicationStatus}> + @httpPayload + Results: Document +} diff --git a/models/backbeatApis/scheduleIngestionSiteResume.smithy b/models/backbeatApis/scheduleIngestionSiteResume.smithy new file mode 100644 index 00000000..6ff2a271 --- /dev/null +++ b/models/backbeatApis/scheduleIngestionSiteResume.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Schedule a resume for a specific ingestion site +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}/schedule") +operation ScheduleIngestionSiteResume { + input: ScheduleIngestionSiteResumeInput, + output: ScheduleIngestionSiteResumeOutput +} + +structure ScheduleIngestionSiteResumeInput { + /// Site name to schedule resume for + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure ScheduleIngestionSiteResumeOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/models/backbeatApis/scheduleSiteResume.smithy b/models/backbeatApis/scheduleSiteResume.smithy new file mode 100644 index 00000000..6f30950a --- /dev/null +++ b/models/backbeatApis/scheduleSiteResume.smithy @@ -0,0 +1,25 @@ +$version: "2.0" +namespace cloudserver.client + +/// Schedule a resume for a specific Cross-Region Replication (CRR) site +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}/schedule") +operation ScheduleSiteResume { + input: ScheduleSiteResumeInput, + output: ScheduleSiteResumeOutput +} + +structure ScheduleSiteResumeInput { + /// Site name to schedule resume for + @required + @httpLabel + Site: String, + + /// Optional request body + @httpPayload + Body: Blob +} + +structure ScheduleSiteResumeOutput { + /// Status response map + status: LocationStatusMap +} diff --git a/package.json b/package.json index 1663b506..a7bb49d8 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:raft": "jest tests/testRaftApis.test.ts", "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend", "test:metadata-backend": "yarn test:api && yarn test:lifecycle && yarn test:metadata && yarn test:raft", + "test:backbeat-apis": "yarn jest tests/testBackbeatApis.test.ts", "lint": "eslint src tests", "typecheck": "tsc --noEmit" }, diff --git a/service/cloudserver.smithy b/service/cloudserver.smithy index 607f87cb..758b0817 100644 --- a/service/cloudserver.smithy +++ b/service/cloudserver.smithy @@ -39,5 +39,23 @@ service cloudserver { PutBucketIndexes, PutData, PutMetadata, + + // Backbeat Apis + CheckConnection, + GetFailedObject, + GetLocationsIngestionStatus, + GetLocationsStatus, + ListFailed, + PauseAllIngestionSites, + PauseAllSites, + PauseIngestionSite, + PauseSite, + ResumeAllIngestionSites, + ResumeAllSites, + ResumeIngestionSite, + ResumeSite, + RetryFailedObjects, + ScheduleIngestionSiteResume, + ScheduleSiteResume, ] } diff --git a/tests/testBackbeatApis.test.ts b/tests/testBackbeatApis.test.ts new file mode 100644 index 00000000..805cd9fa --- /dev/null +++ b/tests/testBackbeatApis.test.ts @@ -0,0 +1,192 @@ +import { + CloudserverClient, + CheckConnectionInput, + CheckConnectionCommand, + GetLocationsStatusInput, + GetLocationsStatusCommand, + GetLocationsIngestionStatusInput, + GetLocationsIngestionStatusCommand, + ListFailedCommandInput, + ListFailedCommand, + GetFailedObjectInput, + GetFailedObjectCommand, + RetryFailedObjectsCommand, + PauseAllSitesInput, + PauseAllSitesCommand, + PauseAllIngestionSitesInput, + PauseAllIngestionSitesCommand, + PauseSiteInput, + PauseSiteCommand, + PauseIngestionSiteInput, + PauseIngestionSiteCommand, + ResumeAllSitesInput, + ResumeAllSitesCommand, + ResumeAllIngestionSitesInput, + ResumeAllIngestionSitesCommand, + ResumeSiteInput, + ResumeSiteCommand, + ResumeIngestionSiteInput, + ResumeIngestionSiteCommand, + ScheduleSiteResumeInput, + ScheduleSiteResumeCommand, + ScheduleIngestionSiteResumeInput, + ScheduleIngestionSiteResumeCommand, +} from '../src/index'; +import { createTestClient, testConfig } from './testSetup'; +import assert from 'assert'; + +// Note : these tests are relatively simple, as it's not straightforward to setup +// backbeat with locations, and real pause/resume tests scenarios. +// The tests in Zenko are also using these apis, and will represent an opportunity to +// test this client better : https://scality.atlassian.net/browse/ZENKO-5102 +describe('CloudServer Check Status API Tests', () => { + let client: CloudserverClient; + + beforeAll(() => { + ({client} = createTestClient()); + }); + + it('should test checkConnection', async () => { + const input: CheckConnectionInput = {}; + const cmd = new CheckConnectionCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test GetLocationsStatus', async () => { + const input: GetLocationsStatusInput = {}; + const cmd = new GetLocationsStatusCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test GetLocationsIngestionStatus', async () => { + const input: GetLocationsIngestionStatusInput = {}; + const cmd = new GetLocationsIngestionStatusCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test listFailed', async () => { + const input: ListFailedCommandInput = {}; + const cmd = new ListFailedCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test getFailedObject', async () => { + const input: GetFailedObjectInput = { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + }; + const cmd = new GetFailedObjectCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test retryFailedObjects', async () => { + const bodyData = JSON.stringify([ + { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + StorageClass: 'STANDARD', + } + ]); + const cmd = new RetryFailedObjectsCommand({ + Body: new TextEncoder().encode(bodyData) + }); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllSites', async () => { + const input: PauseAllSitesInput = {}; + const cmd = new PauseAllSitesCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllIngestionSites', async () => { + const input: PauseAllIngestionSitesInput = {}; + const cmd = new PauseAllIngestionSitesCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseSite', async () => { + const input: PauseSiteInput = { + Site: 'zenko', // Value from backbeat default config.json, destination.bootstrapList + }; + const cmd = new PauseSiteCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test pauseIngestionSite', async () => { + const input: PauseIngestionSiteInput = { + Site: 'a-zenko-location', // Value from backbeat default config.json, extensions.ingestion.sources + }; + const cmd = new PauseIngestionSiteCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllSites', async () => { + const input: ResumeAllSitesInput = {}; + const cmd = new ResumeAllSitesCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllIngestionSites', async () => { + const input: ResumeAllIngestionSitesInput = {}; + const cmd = new ResumeAllIngestionSitesCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeSite', async () => { + const input: ResumeSiteInput = { + Site: 'zenko', + }; + const cmd = new ResumeSiteCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test resumeIngestionSite', async () => { + const input: ResumeIngestionSiteInput = { + Site: 'a-zenko-location', + }; + const cmd = new ResumeIngestionSiteCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test scheduleSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleSiteResumeInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleSiteResumeCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test scheduleIngestionSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleIngestionSiteResumeInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleIngestionSiteResumeCommand(input); + const result = await client.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); +}); diff --git a/tests/testSetup.ts b/tests/testSetup.ts index d2b338b4..751952bf 100644 --- a/tests/testSetup.ts +++ b/tests/testSetup.ts @@ -48,6 +48,7 @@ export const testConfig = { objectKey: `test-cloudserverclient-object-sla/sh${randomId()}`, objectData: 'iAmSomeData', canonicalID: '39383234313039353433383937313939393939395247303031202036353034352e30', + versionID: '' }; async function initBucketForTests() { @@ -70,7 +71,8 @@ async function initBucketForTests() { Key: testConfig.objectKey, Body: testConfig.objectData, }); - await s3client.send(putObjectCommand); + const result = await s3client.send(putObjectCommand); + testConfig.versionID = result.VersionId!; } catch (error: any) { assert.fail(`Failed to initialize bucket for tests: ${error}`); }