Ludwig/ci utils (SorellaLabs#779)

* test

* wip

* updated cex-dex quotes filtering

* Clippy

* test

* test

* testing for final run

* need more ram ffs

* test

* test

* test

* test

* test

* test kek

* pushing for run

* test

* test

* moved test db to brontes-ci folder to mitigate permission problem

* Updated cli docs & fixing CI

* test

* testing perms issue

* test cleanup for branch name with

/

---------

Co-authored-by: jnoorchashm37 <[email protected]>

Author: 0xvanbeethoven

.github/workflows/bench.yml

@@ -37,13 +37,23 @@ jobs:
           echo "${{ secrets.SSH_CONFIG }}" > ~/.ssh/config
 
       - name: cargo bench
-        run: ssh brontes_server -tt "bash --login -c 'source ~/.bashrc && mkdir -p ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }} && cd ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }} && git clone https://github.com/SorellaLabs/brontes.git && cd brontes && git pull && git checkout ${{ steps.branch-names.outputs.current_branch }} && ./scripts/run-tests.sh ${{ steps.branch-names.outputs.current_branch }} ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }} bench ${{ matrix.features }} '"
+        run: |
+          ssh brontes_server -tt "bash --login -c '
+            source ~/.bashrc
+            mkdir -p ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }}
+            cd ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }}
+            git clone https://github.com/SorellaLabs/brontes.git
+            cd brontes
+            git pull
+            git checkout ${{ steps.branch-names.outputs.current_branch }}
+            ./scripts/run-tests.sh ${{ steps.branch-names.outputs.current_branch }} ${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }} bench ${{ matrix.features }}
+          '"
 
       - name: Cleanup on cancellation
         if: cancelled()
         run: |
           ssh brontes_server -tt "bash --login -c 'source ~/.bashrc
             set -x
             rm -rf /home/brontes-ci/${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }}
-            rm -rf /home/data/brontes-ci/${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }}
+            rm -rf /home/brontes-ci/data/${{ steps.branch-names.outputs.current_branch }}-bench-${{ matrix.features }}
           '"

.github/workflows/ci.yml

@@ -65,9 +65,8 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
-    if: always()
-    #&& !cancelled() && !contains(needs.*.result, 'failure') && github.event.pull_request.draft == false
-    #needs: [clippy, build]
+    if: always() && !cancelled() && !contains(needs.*.result, 'failure') && github.event.pull_request.draft == false
+    needs: [clippy, build]
     strategy:
       fail-fast: false
       matrix:
@@ -96,6 +95,7 @@ jobs:
             cd ${{ steps.branch-names.outputs.current_branch }}-test-${{ matrix.features }}
             git clone https://github.com/SorellaLabs/brontes.git
             cd brontes
+            git checkout ${{ steps.branch-names.outputs.current_branch }}
             ./scripts/run-tests.sh ${{ steps.branch-names.outputs.current_branch }} ${{ steps.branch-names.outputs.current_branch }}-test-${{ matrix.features }} test ${{ matrix.features }}
           '"
       - name: Cleanup
@@ -104,7 +104,7 @@ jobs:
           ssh brontes_server -tt "bash --login -c '
             set -x
             rm -rf \$HOME/${{ steps.branch-names.outputs.current_branch }}-test-${{ matrix.features }}
-            rm -rf /home/data/brontes-ci/${{ steps.branch-names.outputs.current_branch }}-test-${{ matrix.features }}
+            rm -rf /home/brontes-ci/data/${{ steps.branch-names.outputs.current_branch }}-test-${{ matrix.features }}
           '"
 
       - name: Check test result
@@ -143,6 +143,7 @@ jobs:
             cd ${{ steps.branch-names.outputs.current_branch }}-it-${{ matrix.features }}
             git clone https://github.com/SorellaLabs/brontes.git
             cd brontes
+            git checkout ${{ steps.branch-names.outputs.current_branch }}
             ./scripts/run-tests.sh ${{ steps.branch-names.outputs.current_branch }} ${{ steps.branch-names.outputs.current_branch }}-it-${{ matrix.features }} it ${{ matrix.features }}
           '"
 
@@ -152,9 +153,32 @@ jobs:
           ssh brontes_server -tt "bash --login -c '
             set -x
             rm -rf \$HOME/${{ steps.branch-names.outputs.current_branch }}-it-${{ matrix.features }}
-            rm -rf /home/data/brontes-ci/${{ steps.branch-names.outputs.current_branch }}-it-${{ matrix.features }}
+            rm -rf /home/brontes-ci/data/${{ steps.branch-names.outputs.current_branch }}-it-${{ matrix.features }}
           '"
 
       - name: Check integration test result
         if: steps.run_it_tests.outcome == 'failure'
         run: exit 1
+
+  cleanup:
+    name: "Cleanup"
+    needs: [test, it-test]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Init ssh config
+        run: |
+          mkdir ~/.ssh/ 
+          echo "${{ secrets.BASTION_SSH }}" > ~/.ssh/bastion 
+          chmod 600 ~/.ssh/bastion 
+          echo "${{ secrets.BRONTES_SSH }}" > ~/.ssh/brontes_server 
+          chmod 600 ~/.ssh/brontes_server
+          echo "${{ secrets.SSH_CONFIG }}" > ~/.ssh/config
+
+      - name: Run cleanup
+        run: |
+          ssh brontes_server -tt "bash --login -c '
+            set -x
+            rm -rf \$HOME/*/
+            echo \"Cleanup completed\"
+          '"