Skip CI workflows based on the list of modified files (vitessio#10031)

* Try paths-filter Signed-off-by: Sara Bee <[email protected]> Signed-off-by: Florent Poinsard <[email protected]> * forgot 'uses' Signed-off-by: Sara Bee <[email protected]> * Regenerate workflows Signed-off-by: Sara Bee <[email protected]> * feat: addition of skip paths in every workflow templates Signed-off-by: Florent Poinsard <[email protected]> * feat: re generated CI files Signed-off-by: Florent Poinsard <[email protected]> * feat: filter end-to-end tests per **/endtoend/**.go Signed-off-by: Florent Poinsard <[email protected]> * feat: fix end-to-end filter and addition of filter for go_files Signed-off-by: Florent Poinsard <[email protected]> * feat: comprehensive filtering for end-to-end tests Signed-off-by: Florent Poinsard <[email protected]> * feat: filter go files for golint, gofmt, and goimports Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of path filtering for unit test workflows Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of path filtering for check make parser Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of bootstrap.sh to the path filtering and new filter for make proto Signed-off-by: Florent Poinsard <[email protected]> * feat: path filtering for make sizegen Signed-off-by: Florent Poinsard <[email protected]> * feat: path filtering for make visitor Signed-off-by: Florent Poinsard <[email protected]> * feat: path filtering for unit_race Signed-off-by: Florent Poinsard <[email protected]> * feat: fix condition for check make proto Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of bootstrap to the unit_test workflows Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of path filtering for end-to-end, examples workflows, ugprade downgrade workflows Signed-off-by: Florent Poinsard <[email protected]> * feat: addition of path filtering for docker and initial sharding multi workflows Signed-off-by: Florent Poinsard <[email protected]> * feat: revert test change in misc_test.go Signed-off-by: Florent Poinsard <[email protected]> Co-authored-by: Sara Bee <[email protected]>
github · Apr 12, 2022 · 5edbc68 · 5edbc68
1 parent 9c082bd
commit 5edbc68
Show file tree

Hide file tree

Showing 96 changed files with 2,439 additions and 333 deletions.
diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml
@@ -1,30 +1,43 @@
 name: check_formatting
 on: [pull_request,push]
 jobs:
-
   build:
     name: Check Formatting
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      if: needs.check_relevant_files.outputs.go_files == 'true'
+      uses: actions/checkout@v2
+
+    - name: Check for changes in Go files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          go_files:
+            - '**/*.go'
+            - '*.go'
+            - 'go.[sumod]'
 
     - name: Set up Go
+      if: steps.changes.outputs.go_files == 'true'
       uses: actions/setup-go@v2
       with:
         go-version: 1.18
 
     - name: Tune the OS
+      if: steps.changes.outputs.go_files == 'true'
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
     # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
     - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      if: steps.changes.outputs.go_files == 'true'
       run: |
         echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
     # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
     - name: Run go fmt
+      if: steps.changes.outputs.go_files == 'true'
       run: |
         gofmt -l . | grep -vF vendor/ && exit 1 || echo "All files formatted correctly"
diff --git a/.github/workflows/check_imports.yml b/.github/workflows/check_imports.yml
@@ -1,25 +1,36 @@
 name: check_imports
 on: [pull_request,push]
 jobs:
-
   build:
     name: Check Imports
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Check for changes in Go files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          go_files:
+            - '**/*.go'
+            - '*.go'
+            - 'go.[sumod]'
 
     - name: Set up Go
+      if: steps.changes.outputs.go_files == 'true'
       uses: actions/setup-go@v2
       with:
         go-version: 1.18
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
     - name: Install goimports
+      if: steps.changes.outputs.go_files == 'true'
       run: |
         go install golang.org/x/tools/cmd/goimports@latest
 
     - name: Run goimports
+      if: steps.changes.outputs.go_files == 'true'
       run: |
         out=$(goimports -local vitess.io/vitess -l -w $(find . -name "*.go" | grep -v ".pb.go"))
         echo $out | grep go > /dev/null && echo -e "The following files are malformatted:\n$out" && exit 1 || echo "All the files are formatted correctly"
diff --git a/.github/workflows/check_make_parser.yml b/.github/workflows/check_make_parser.yml
@@ -6,30 +6,46 @@ jobs:
     name: Check Make Parser
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Check for changes in relevant files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          parser_changes:
+            - 'go/vt/sqlparser/**'
+            - 'go.[sumod]'
+            - 'build.env'
+            - 'bootstrap.sh'
+            - 'tools/**'
 
     - name: Set up Go
       uses: actions/setup-go@v2
+      if: steps.changes.outputs.parser_changes == 'true'
       with:
         go-version: 1.18
 
     - name: Tune the OS
+      if: steps.changes.outputs.parser_changes == 'true'
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
     - name: Get dependencies
+      if: steps.changes.outputs.parser_changes == 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y make unzip g++ etcd curl git wget
         sudo service etcd stop
         go mod download
 
     - name: Run make minimaltools
+      if: steps.changes.outputs.parser_changes == 'true'
       run: |
         make minimaltools
 
     - name: check_make_parser
+      if: steps.changes.outputs.parser_changes == 'true'
       run: |
         tools/check_make_parser.sh
diff --git a/.github/workflows/check_make_proto.yml b/.github/workflows/check_make_proto.yml
@@ -6,20 +6,37 @@ jobs:
     name: Check Make Proto
     runs-on: ubuntu-latest
     steps:
-
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Check for changes in relevant files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          proto_changes:
+            - 'bootstrap.sh'
+            - 'tools/**'
+            - 'build.env'
+            - 'go.[sumod]'
+            - 'Makefile'
+            - 'go/vt/proto/**'
+            - 'proto/*.proto'
+
     - name: Set up Go
       uses: actions/setup-go@v2
+      if: steps.changes.outputs.proto_changes == 'true'
       with:
         go-version: 1.18
 
     - name: Tune the OS
+      if: steps.changes.outputs.proto_changes == 'true'
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    - name: Check out code
-      uses: actions/checkout@v2
 
     - name: Get dependencies
+      if: steps.changes.outputs.proto_changes == 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y make unzip g++ etcd curl git wget
@@ -28,9 +45,11 @@ jobs:
         go install golang.org/x/tools/cmd/goimports@latest
 
     - name: Run make minimaltools
+      if: steps.changes.outputs.proto_changes == 'true'
       run: |
         make minimaltools
 
     - name: check_make_proto
+      if: steps.changes.outputs.proto_changes == 'true'
       run: |
         tools/check_make_proto.sh
diff --git a/.github/workflows/check_make_sizegen.yml b/.github/workflows/check_make_sizegen.yml
@@ -6,20 +6,37 @@ jobs:
     name: Check Make Sizegen
     runs-on: ubuntu-latest
     steps:
-
+    - name: Check for changes in relevant files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          sizegen:
+            - 'go/**/*.go'
+            - 'test.go'
+            - 'Makefile'
+            - 'build.env'
+            - 'go.[sumod]'
+            - 'tools/**'
+            - 'bootstrap.sh'
+
     - name: Set up Go
+      if: steps.changes.outputs.sizegen == 'true'
       uses: actions/setup-go@v2
       with:
         go-version: 1.18
 
     - name: Tune the OS
+      if: steps.changes.outputs.sizegen == 'true'
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
     - name: Check out code
+      if: steps.changes.outputs.sizegen == 'true'
       uses: actions/checkout@v2
 
     - name: Get dependencies
+      if: steps.changes.outputs.sizegen == 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y make unzip g++ etcd curl git wget
@@ -28,9 +45,11 @@ jobs:
         go install golang.org/x/tools/cmd/goimports@latest
 
     - name: Run make minimaltools
+      if: steps.changes.outputs.sizegen == 'true'
       run: |
         make minimaltools
 
     - name: check_make_sizegen
+      if: steps.changes.outputs.sizegen == 'true'
       run: |
         tools/check_make_sizegen.sh
diff --git a/.github/workflows/check_make_visitor.yml b/.github/workflows/check_make_visitor.yml
@@ -6,20 +6,37 @@ jobs:
     name: Check Make Visitor
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Check for changes in relevant files
+      uses: dorny/paths-filter@v2
+      id: changes
+      with:
+        filters: |
+          visitor:
+            - 'go/tools/asthelpergen/**'
+            - 'go/vt/sqlparser/**'
+            - 'Makefile'
+            - 'build.env'
+            - 'go.[sumod]'
+            - 'tools/**'
+            - 'bootstrap.sh'
+            - 'misc/git/hooks/asthelpers'
 
     - name: Set up Go
+      if: steps.changes.outputs.visitor == 'true'
       uses: actions/setup-go@v2
       with:
         go-version: 1.18
 
     - name: Tune the OS
+      if: steps.changes.outputs.visitor == 'true'
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
     - name: Get dependencies
+      if: steps.changes.outputs.visitor == 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y make unzip g++ etcd curl git wget
@@ -28,9 +45,11 @@ jobs:
         go install golang.org/x/tools/cmd/goimports@latest
 
     - name: Run make minimaltools
+      if: steps.changes.outputs.visitor == 'true'
       run: |
         make minimaltools
 
     - name: check_make_visitor
+      if: steps.changes.outputs.visitor == 'true'
       run: |
         misc/git/hooks/asthelpers
diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml
@@ -15,27 +15,48 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
+      - name: Check for changes in relevant files
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            end_to_end:
+              - 'go/**/*.go'
+              - 'test.go'
+              - 'Makefile'
+              - 'build.env'
+              - 'go.[sumod]'
+              - 'proto/*.proto'
+              - 'tools/**'
+              - 'config/**'
+              - '.github/docker/**'
+              - 'bootstrap.sh'
+
       - name: Build Docker Image
+        if: steps.changes.outputs.end_to_end == 'true'
         run: docker build -f ./.github/docker/cluster_test_12/Dockerfile -t cluster_test_12:$GITHUB_SHA  .
 
       - name: Run test
+        if: steps.changes.outputs.end_to_end == 'true'
         timeout-minutes: 30
         run: docker run --name "cluster_test_12_$GITHUB_SHA" cluster_test_12:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard 12 -- -- --keep-data=true'
 
       - name: Print Volume Used
-        if: ${{ always() }}
+        if: always() && steps.changes.outputs.end_to_end == 'true'
         run: |
           docker inspect -f '{{ (index .Mounts 0).Name }}' cluster_test_12_$GITHUB_SHA
 
       - name: Cleanup Docker Volume
+        if: steps.changes.outputs.end_to_end == 'true'
         run: |
           docker rm -v cluster_test_12_$GITHUB_SHA
 
       - name: Cleanup Docker Container
-        if: ${{ always() }}
+        if: always() && steps.changes.outputs.end_to_end == 'true'
         run: |
           docker rm -f cluster_test_12_$GITHUB_SHA
 
       - name: Cleanup Docker Image
+        if: steps.changes.outputs.end_to_end == 'true'
         run: |
           docker image rm cluster_test_12:$GITHUB_SHA