From 61453df503e9361257bccb13305075340884fff4 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 05:38:13 +0000 Subject: [PATCH 1/8] Update the buildAndTestCMake: save core dumps Save core dumps for buildAndTestCMake so that we can triage failures that occur. Co-authored-by: Mark Williams --- .github/workflows/buildAndTestCMake.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index caad5aa114..8b9cac8456 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -73,11 +73,16 @@ jobs: env: CMAKE_BUILD_TYPE: Release MLIR_ENABLE_BINDINGS_PYTHON: ON - + - name: Setup Core Dumps + run: | + # e is the process name, p is the process id + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited - name: Build and Test StableHLO (with AddressSanitizer) shell: bash run: | - ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" + ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) + sudo ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" env: CMAKE_BUILD_TYPE: Release STABLEHLO_ENABLE_BINDINGS_PYTHON: OFF @@ -89,4 +94,12 @@ jobs: ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" env: CMAKE_BUILD_TYPE: Release - STABLEHLO_ENABLE_BINDINGS_PYTHON: ON \ No newline at end of file + STABLEHLO_ENABLE_BINDINGS_PYTHON: ON + + - name: Upload Core Dumps + if: always() + uses: actions/upload-artifact@v4 + with: + name: crashes + path: core* + if-no-files-found: ignore \ No newline at end of file From 362c577a69a5de575557f778bb15ab330304b4df Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 05:44:42 +0000 Subject: [PATCH 2/8] whitespace fix --- .github/workflows/buildAndTestCMake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 8b9cac8456..5b9528f441 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -81,7 +81,7 @@ jobs: - name: Build and Test StableHLO (with AddressSanitizer) shell: bash run: | - ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) + ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) sudo ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" env: CMAKE_BUILD_TYPE: Release @@ -95,11 +95,11 @@ jobs: env: CMAKE_BUILD_TYPE: Release STABLEHLO_ENABLE_BINDINGS_PYTHON: ON - + - name: Upload Core Dumps if: always() uses: actions/upload-artifact@v4 with: name: crashes path: core* - if-no-files-found: ignore \ No newline at end of file + if-no-files-found: ignore From fde7632144fea6b5c9b5157c6fb9f272c741ad7f Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 05:57:43 +0000 Subject: [PATCH 3/8] minor cleanup --- .github/workflows/buildAndTestCMake.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 5b9528f441..45896694da 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -77,12 +77,11 @@ jobs: run: | # e is the process name, p is the process id sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - name: Build and Test StableHLO (with AddressSanitizer) shell: bash run: | ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) - sudo ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" + ./build_tools/github_actions/ci_build_cmake.sh "$LLVM_BUILD_DIR" "$STABLEHLO_BUILD_DIR" env: CMAKE_BUILD_TYPE: Release STABLEHLO_ENABLE_BINDINGS_PYTHON: OFF From 069948cf94da821a0ac2312fbb4fea8e0bfbb98f Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 06:17:16 +0000 Subject: [PATCH 4/8] Add printing filesystem usage --- .github/workflows/buildAndTestCMake.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 45896694da..919b300a26 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -95,6 +95,9 @@ jobs: CMAKE_BUILD_TYPE: Release STABLEHLO_ENABLE_BINDINGS_PYTHON: ON + - name: Print Filesystem Usage + run: df -H + - name: Upload Core Dumps if: always() uses: actions/upload-artifact@v4 From cb4339690c02eefe149f1087f6f82003e3377c4a Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 06:23:29 +0000 Subject: [PATCH 5/8] Add always to printing disk usage --- .github/workflows/buildAndTestCMake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 919b300a26..198b773bd0 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -96,6 +96,7 @@ jobs: STABLEHLO_ENABLE_BINDINGS_PYTHON: ON - name: Print Filesystem Usage + if: always() run: df -H - name: Upload Core Dumps From b0ae590a98c2e4213ecac92e8c6eff50fe8375b0 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 16:55:11 +0000 Subject: [PATCH 6/8] Moved to /cores directory --- .github/workflows/buildAndTestCMake.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 198b773bd0..4151985cdd 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -75,8 +75,10 @@ jobs: MLIR_ENABLE_BINDINGS_PYTHON: ON - name: Setup Core Dumps run: | + sudo mkdir /cores + sudo chmod 777 /cores # e is the process name, p is the process id - sudo sysctl -w kernel.core_pattern="core.%e.%p" + sudo sysctl -w kernel.core_pattern="/cores/core.%e.%p" - name: Build and Test StableHLO (with AddressSanitizer) shell: bash run: | @@ -95,14 +97,10 @@ jobs: CMAKE_BUILD_TYPE: Release STABLEHLO_ENABLE_BINDINGS_PYTHON: ON - - name: Print Filesystem Usage - if: always() - run: df -H - - name: Upload Core Dumps if: always() uses: actions/upload-artifact@v4 with: name: crashes - path: core* + path: /cores/* if-no-files-found: ignore From 3d4591d00085143dcd5d2a640100b3a5bfaae8ca Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 17:08:29 +0000 Subject: [PATCH 7/8] verify coredumps work --- .github/workflows/buildAndTestCMake.yml | 21 +++++++++++++-------- x.c | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 x.c diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 4151985cdd..51788d6f0b 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -65,7 +65,18 @@ jobs: uses: ./.github/actions/setup-build with: llvm-version: ${{ steps.llvm-version.outputs.version }} - + - name: Setup Core Dumps + run: | + sudo mkdir /cores + sudo chmod 777 /cores + # Core filenames will be of the form executable.pid.timestamp: + sudo bash -c 'echo "/cores/%e.%p.%t" > /proc/sys/kernel/core_pattern' + - name: Build and Run X + run: | + ulimit -c unlimited + make x + sudo ./x + ./x - name: Configure and Build LLVM shell: bash run: | @@ -73,12 +84,6 @@ jobs: env: CMAKE_BUILD_TYPE: Release MLIR_ENABLE_BINDINGS_PYTHON: ON - - name: Setup Core Dumps - run: | - sudo mkdir /cores - sudo chmod 777 /cores - # e is the process name, p is the process id - sudo sysctl -w kernel.core_pattern="/cores/core.%e.%p" - name: Build and Test StableHLO (with AddressSanitizer) shell: bash run: | @@ -98,7 +103,7 @@ jobs: STABLEHLO_ENABLE_BINDINGS_PYTHON: ON - name: Upload Core Dumps - if: always() + if: ${{ failure() }} uses: actions/upload-artifact@v4 with: name: crashes diff --git a/x.c b/x.c new file mode 100644 index 0000000000..2cb27d9b44 --- /dev/null +++ b/x.c @@ -0,0 +1,19 @@ +#include +#include + +int x() { + return 1; +} + +int y() { + return 1 + x(); +} + +int z() { + return 2 + y(); +} + +int main() { + kill(getpid(), SIGABRT); + return 3 + z(); +} From 5266df1e4feab25c8f5207736b82e79e6fa3a9cd Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 16 Mar 2024 17:18:19 +0000 Subject: [PATCH 8/8] cat kernel_pattern to verify --- .github/workflows/buildAndTestCMake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildAndTestCMake.yml b/.github/workflows/buildAndTestCMake.yml index 51788d6f0b..616200721f 100644 --- a/.github/workflows/buildAndTestCMake.yml +++ b/.github/workflows/buildAndTestCMake.yml @@ -75,7 +75,7 @@ jobs: run: | ulimit -c unlimited make x - sudo ./x + cat /proc/sys/kernel/core_pattern ./x - name: Configure and Build LLVM shell: bash