feat: components tests on local cluster (#3457)

Tests partitions behavior through Fluvio CLI by producing and then consuming
with specific partitions.

.github/workflows/ci.yml

@@ -1091,6 +1091,106 @@ jobs:
           path: diagnostics*.gz
           retention-days: 1
 
+  # Runs tests on `tests/cli/partition_test`
+  partition_test:
+    name: Partitions Test (${{ matrix.test }} with ${{ matrix.spu }} SPUs, ${{ matrix.partitions }} Partitions and ${{ matrix.replication }} Replicas) on ${{ matrix.os }} for ${{ matrix.rust-target }}
+    runs-on: ${{ matrix.os }}
+    env:
+      FLUVIO_BIN: "~/bin/fluvio"
+      SERVER_LOG: fluvio=info
+    needs:
+      - build_primary_binaries
+      - config
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        rust-target:
+          - x86_64-unknown-linux-musl
+        test:
+          - multiple_partitions
+        spu:
+          - 2
+        partitions:
+          - 2
+          - 4
+          - 6
+        replication:
+          - 1
+    steps:
+      - name: Checkout Source Code
+        uses: actions/checkout@v3
+
+      # Download Artifacts from Development Build (This Commit)
+      - name: Download artifact - fluvio
+        uses: actions/download-artifact@v3
+        with:
+          name: fluvio-${{ matrix.rust-target }}
+          path: ~/bin
+
+      - name: Download artifact - fluvio-run
+        uses: actions/download-artifact@v3
+        with:
+          name: fluvio-run-${{ matrix.rust-target }}
+          path: ~/.fluvio/extensions
+
+      - name: Setup K8s Cluster
+        run: |
+          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=${{ env.K3D_VERSION }} bash
+          ./k8-util/cluster/reset-k3d.sh
+
+      - name: Wait 15s for K3D Reset
+        run: sleep 15
+
+      - name: Set up Fluvio Binaries
+        run: |
+          chmod +x ~/bin/fluvio ~/.fluvio/extensions/fluvio-run
+          echo "~/bin" >> $GITHUB_PATH
+          FLUVIO_BIN=~/bin/fluvio
+          echo "FLUVIO_BIN=${FLUVIO_BIN}" >> $GITHUB_ENV
+
+      - name: Print version
+        run: fluvio version
+
+      - name: Start fluvio cluster
+        timeout-minutes: 10
+        run: fluvio cluster start --local --spu ${{ matrix.spu }}
+
+      - name: Await Cluster Startup
+        run: sleep 15
+
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
+        with:
+          bats-version: ${{ env.BATS_VERSION }}
+
+      # Environment setup is ready, time to run the tests
+      - name: Run Multiple Partitions Test
+        if: matrix.test == 'multiple_partitions'
+        timeout-minutes: 15
+        env:
+          REPLICATION: ${{ matrix.replication }}
+          PARTITIONS: ${{ matrix.partitions }}
+        run: make cli-partition-test-multiple-partitions
+
+      - name: Shutdown Fluvio cluster
+        timeout-minutes: 10
+        run: fluvio cluster shutdown --local
+
+      - name: Run diagnostics
+        if: ${{ !success() }}
+        timeout-minutes: 5
+        run: fluvio cluster diagnostics --local
+
+      - name: Upload diagnostics
+        uses: actions/upload-artifact@v3
+        timeout-minutes: 5
+        if: ${{ !success() }}
+        with:
+          name: fluvio-components-${{ matrix.run }}-${{ matrix.test }}-diag
+          path: diagnostics*.gz
+          retention-days: 1
+
   # Ensure all checks, tests are perform and all binaries are built
   # After this, we are committed for release
   docker_push:
@@ -1107,6 +1207,7 @@ jobs:
       - k8_upgrade_test
       - cli_smoke
       - build_binaries
+      - partition_test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

makefiles/test.mk

@@ -196,6 +196,9 @@ endif
 cli-platform-cross-version-test:
 	bats -t ./tests/cli/cli-platform-cross-version.bats
 
+cli-partition-test-multiple-partitions:
+	bats ./tests/cli/partition_test/multiple_partitions.bats
+
 cli-fluvio-smoke:
 	bats $(shell ls -1 ./tests/cli/fluvio_smoke_tests/*.bats | sort -R)
 	bats ./tests/cli/fluvio_smoke_tests/non-concurrent/cluster-delete.bats

tests/cli/partition_test/multiple_partitions.bats

@@ -0,0 +1,63 @@
+#!/usr/bin/env bats
+
+TEST_HELPER_DIR="$BATS_TEST_DIRNAME/../test_helper"
+export TEST_HELPER_DIR
+
+load "$TEST_HELPER_DIR"/tools_check.bash
+load "$TEST_HELPER_DIR"/fluvio_dev.bash
+load "$TEST_HELPER_DIR"/bats-support/load.bash
+load "$TEST_HELPER_DIR"/bats-assert/load.bash
+
+setup_file() {
+    PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME=$(random_string)
+    export PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME
+    debug_msg "Topic name: $PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME"
+
+    MULTI_LINE_FILE_NAME=$(random_string)
+    export MULTI_LINE_FILE_NAME
+
+    # Creates test File which will have 2 items per partition
+    for (( p = 0; p < $PARTITIONS * 2; p++ ))
+    do
+        echo $p >> "$MULTI_LINE_FILE_NAME"
+    done
+}
+
+teardown_file() {
+    echo "Tearing down, shutting down cluster components"
+    "$FLUVIO_BIN" topic delete "$PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME"
+}
+
+@test "Create a topic for P/C Multiple Partitions" {
+    echo "Creates Topic: $PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME for P/C Multiple Partitions"
+    run timeout 15s "$FLUVIO_BIN" topic create "$PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME" --partitions $PARTITIONS --replication $REPLICATION
+    assert_success
+
+    echo "Topic Details: $PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME"
+    run timeout 15s "$FLUVIO_BIN" topic describe "$PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME"
+    assert_success
+}
+
+@test "Produces on topic for P/C Multiple Partitions" {
+    run bash -c 'timeout 15s "$FLUVIO_BIN" produce --file "$MULTI_LINE_FILE_NAME" "$PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME"'
+    assert_success
+}
+
+@test "Consumes on topic for P/C Multiple Partitions with Partition" {
+    for (( part = 0; part < $PARTITIONS; part++ ))
+    do
+        run timeout 15s "$FLUVIO_BIN" consume "$PRODUCE_CONSUME_MULTIPLE_PARTITIONS_TOPIC_NAME" -p "$part" -B -d
+
+        for set in {0..1}
+        do
+            if (( $set == 0 ))
+            then
+                WANT=$(( $part ))
+                assert_line --index 0 "$WANT"
+            else
+                WANT=$(( $PARTITIONS + $part ))
+                assert_line --index 1 "$WANT"
+            fi
+        done
+    done
+}