diff --git a/.github/workflows/code_coverage.yml b/.github/workflows/code_coverage.yml index 992a7e86cb1..276f2841312 100644 --- a/.github/workflows/code_coverage.yml +++ b/.github/workflows/code_coverage.yml @@ -258,10 +258,10 @@ jobs: evaluate_code_coverage_reports: name: Evaluate Code Coverage Reports runs-on: ubuntu-20.04 - needs: code_coverage_run + needs: [ check_unit_tests_completed, code_coverage_run ] # The expression if: ${{ !cancelled() }} runs a job or step regardless of its success or failure while responding to cancellations, # serving as a cancellation-compliant alternative to if: ${{ always() }} in concurrent workflows. - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.check_unit_tests_completed.result == 'success'}} env: CACHE_DIRECTORY: ~/.bazel_cache steps: @@ -311,12 +311,16 @@ jobs: # Reference: https://github.community/t/127354/7. check_coverage_results: name: Check Code Coverage Results - needs: [ compute_changed_files, code_coverage_run, evaluate_code_coverage_reports ] + needs: [ check_unit_tests_completed, compute_changed_files, code_coverage_run, evaluate_code_coverage_reports ] # The expression if: ${{ !cancelled() }} runs a job or step regardless of its success or failure while responding to cancellations, # serving as a cancellation-compliant alternative to if: ${{ always() }} in concurrent workflows. if: ${{ !cancelled() }} runs-on: ubuntu-20.04 steps: + - name: Check unit tests passed + if: ${{ needs.check_unit_tests_completed.result != 'success' }} + run: exit 1 + - name: Check coverages passed if: ${{ needs.compute_changed_files.outputs.can_skip_files != 'true' && needs.code_coverage_run.result != 'success' }} run: exit 1 diff --git a/.github/workflows/comment_coverage_report.yml b/.github/workflows/comment_coverage_report.yml index 9ecb4e2e95c..16c1ae0da63 100644 --- a/.github/workflows/comment_coverage_report.yml +++ b/.github/workflows/comment_coverage_report.yml @@ -3,11 +3,11 @@ name: Comment Coverage Report # Controls when the action will run. Triggers the workflow on pull request events -# (assigned, opened, synchronize, reopened) +# (opened, synchronize, reopened) on: pull_request_target: - types: [assigned, opened, synchronize, reopened] + types: [opened, synchronize, reopened] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/scripts/src/java/org/oppia/android/scripts/coverage/reporter/BUILD.bazel b/scripts/src/java/org/oppia/android/scripts/coverage/reporter/BUILD.bazel index 85b59087cc2..217bc961aa1 100644 --- a/scripts/src/java/org/oppia/android/scripts/coverage/reporter/BUILD.bazel +++ b/scripts/src/java/org/oppia/android/scripts/coverage/reporter/BUILD.bazel @@ -15,5 +15,6 @@ kt_jvm_library( "//scripts/src/java/org/oppia/android/scripts/common:bazel_client", "//scripts/src/java/org/oppia/android/scripts/proto:coverage_java_proto", "//scripts/src/java/org/oppia/android/scripts/proto:script_exemptions_java_proto", + "//third_party:com_google_guava_guava", ], ) diff --git a/scripts/src/java/org/oppia/android/scripts/coverage/reporter/CoverageReporter.kt b/scripts/src/java/org/oppia/android/scripts/coverage/reporter/CoverageReporter.kt index 31763392945..f447a58c256 100644 --- a/scripts/src/java/org/oppia/android/scripts/coverage/reporter/CoverageReporter.kt +++ b/scripts/src/java/org/oppia/android/scripts/coverage/reporter/CoverageReporter.kt @@ -1,5 +1,6 @@ package org.oppia.android.scripts.coverage.reporter +import com.google.common.html.HtmlEscapers import org.oppia.android.scripts.proto.Coverage import org.oppia.android.scripts.proto.CoverageReport import org.oppia.android.scripts.proto.CoverageReportContainer @@ -277,7 +278,7 @@ class CoverageReporter( """ ${lineNumber.toString().padStart(4, ' ')} - $line + ${HtmlEscapers.htmlEscaper().escape(line)} """.trimIndent() ) diff --git a/scripts/src/javatests/org/oppia/android/scripts/coverage/RunCoverageTest.kt b/scripts/src/javatests/org/oppia/android/scripts/coverage/RunCoverageTest.kt index 7ef0183f00d..3d28a816c7d 100644 --- a/scripts/src/javatests/org/oppia/android/scripts/coverage/RunCoverageTest.kt +++ b/scripts/src/javatests/org/oppia/android/scripts/coverage/RunCoverageTest.kt @@ -317,7 +317,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(kotlinFilePath) - assertThat(readHtmlReport(kotlinFilePath)).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(kotlinFilePath) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -342,7 +348,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(sourceFilePath) - assertThat(readHtmlReport(sourceFilePath)).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(sourceFilePath) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -541,7 +553,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePath) - assertThat(readHtmlReport(filePath)).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePath) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -1880,10 +1898,20 @@ class RunCoverageTest { ).execute() val expectedResult1 = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult1) + val unescapedHtmlReport1 = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + assertThat(unescapedHtmlReport1).isEqualTo(expectedResult1) val expectedResult2 = getExpectedHtmlText(filePathList.get(1)) - assertThat(readHtmlReport(filePathList.get(1))).isEqualTo(expectedResult2) + val unescapedHtmlReport2 = readHtmlReport(filePathList.get(1)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + assertThat(unescapedHtmlReport2).isEqualTo(expectedResult2) } @Test @@ -1910,7 +1938,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -1937,7 +1971,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -1964,7 +2004,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -2009,7 +2055,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -2036,7 +2088,13 @@ class RunCoverageTest { val expectedResult = getExpectedHtmlText(filePathList.get(0)) - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test @@ -2262,7 +2320,13 @@ class RunCoverageTest { """.trimIndent() - assertThat(readHtmlReport(filePathList.get(0))).isEqualTo(expectedResult) + val unescapedHtmlReport = readHtmlReport(filePathList.get(0)) + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedHtmlReport).isEqualTo(expectedResult) } @Test diff --git a/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/BUILD.bazel b/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/BUILD.bazel index 6571e76119e..0747162663f 100644 --- a/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/BUILD.bazel +++ b/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/BUILD.bazel @@ -13,6 +13,7 @@ kt_jvm_test( "//scripts/src/java/org/oppia/android/scripts/coverage/reporter:coverage_reporter_lib", "//scripts/src/java/org/oppia/android/scripts/proto:script_exemptions_java_proto", "//testing:assertion_helpers", + "//third_party:com_google_guava_guava", "//third_party:com_google_truth_truth", "//third_party:org_jetbrains_kotlin_kotlin-test-junit", ], diff --git a/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/CoverageReporterTest.kt b/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/CoverageReporterTest.kt index b0af56a1115..ae06e6a693e 100644 --- a/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/CoverageReporterTest.kt +++ b/scripts/src/javatests/org/oppia/android/scripts/coverage/reporter/CoverageReporterTest.kt @@ -697,7 +697,13 @@ class CoverageReporterTest { """.trimIndent() - assertThat(outputReportText).isEqualTo(expectedHtml) + val unescapedOutputReportText = outputReportText + .replace(""", "\"") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + assertThat(unescapedOutputReportText).isEqualTo(expectedHtml) } @Test diff --git a/wiki/Oppia-Android-Code-Coverage.md b/wiki/Oppia-Android-Code-Coverage.md index e5ed0479619..499b3591642 100644 --- a/wiki/Oppia-Android-Code-Coverage.md +++ b/wiki/Oppia-Android-Code-Coverage.md @@ -139,7 +139,7 @@ Coverage Analysis: **FAIL** :x:
| File | Coverage | Lines Hit | Status | Min Required | |------|:--------:|----------:|:------:|:------------:| -|
MathTokenizer.ktutility/src/main/java/org/oppia/android/util/math/MathTokenizer.kt
| 94.26% | 197 / 209 | :white_check_mark: | 70% | +|
Pass.ktutility/src/main/java/org/oppia/android/util/math/Pass.kt
| 94.26% | 197 / 209 | :white_check_mark: | 70% | ### Exempted coverage @@ -282,7 +282,7 @@ Certain files are exempt from coverage checks. These exemptions include: 1. **Test File Exemptions:** Files that are exempted from having corresponding test files are also exempted from coverage checks. Since no test files are available for these sources, coverage analysis cannot be performed, and these files are therefore skipped. -2. **Source File Incompatibility Exemptions:** Some files are currently incompatible with Bazel coverage execution ([see tracking issue #5481](https://github.com/oppia/oppia-android/issues/5481)) and are temporarily excluded from coverage checks. +2. **Source File Incompatibility Exemptions:** Some files are currently incompatible with Bazel coverage execution (see tracking issue [#5481](https://github.com/oppia/oppia-android/issues/5481)) and are temporarily excluded from coverage checks. You can find the complete list of exemptions in this file: [test_file_exemptions.textproto](https://github.com/oppia/oppia-android/blob/develop/scripts/assets/test_file_exemptions.textproto) @@ -311,7 +311,19 @@ bazel run //scripts:run_coverage -- : Your root directory. - : Files you want to generate coverage reports for. -For example, to analyze coverage for the file MathTokenizer.kt, use the relative path: +To get the relative path of a file: + +1. Navigate to the Project view on the left-hand side in Android Studio. +2. Locate the file to analyze Code Coverage for. +3. Right click the file and select Copy Path. To get the path relative to the root. + +Alternatively, the coverage report itself provides the relative paths. You can reveal this information by clicking on the drop-down that precedes the file name in the report. + +| File | Coverage | Lines Hit | Status | Min Required | +|------|:--------:|----------:|:------:|:------------:| +|
MathTokenizer.ktutility/src/main/java/org/oppia/android/util/math/MathTokenizer.kt
| 94.26% | 197 / 209 | :white_check_mark: | 70% | + +To analyze coverage for the file MathTokenizer.kt, use the relative path: ```sh bazel run //scripts:run_coverage -- $(pwd) utility/src/main/java/org/oppia/android/util/math/MathTokenizer.kt @@ -456,6 +468,14 @@ It’s essential to ensure that each source file is directly tested by its own c ## Limitations of the code coverage tool -1. **Incompatibility with Code Coverage Analysis:** Certain test targets in the Oppia-Android codebase fail to execute and collect coverage using the Bazel coverage command. The underlying issues are still being investigated ([see tracking issue #5481](https://github.com/oppia/oppia-android/issues/5481)), and these files are currently exempt from coverage checks. However, it's expected that all new test files should work without needing this exemption. +1. **Incompatibility with Code Coverage Analysis:** Certain test targets in the Oppia-Android codebase fail to execute and collect coverage using the Bazel coverage command. The underlying issues are still being investigated (see tracking issue [#5481](https://github.com/oppia/oppia-android/issues/5481)), and these files are currently exempt from coverage checks. However, it's expected that all new test files should work without needing this exemption. 2. **Function and Branch Coverage:** The Oppia-Android code coverage tool currently provides only line coverage data. It does not include information on function or branch coverage. + +3. **Kotlin inline functions:** With JaCoCo coverage gaps, Kotlin inline functions may be inaccurately reported as uncovered in coverage reports. (See tracking issue [#5501](https://github.com/oppia/oppia-android/issues/5501)) + +4. **Line and Partial Coverages:** The current line coverage analysis in Oppia Android is limited and may not accurately reflect the execution of complex or multi-branch code within a single line, reporting lines as fully covered even if only part of the logic within those lines is executed, leading to potentially misleading coverage data. (See tracking issue [#5503](https://github.com/oppia/oppia-android/issues/5503)) + +5. **Flow Interrupting Statements:** The coverage reports may inaccurately reflect the coverage of flow-interrupting statements (e.g., exitProcess(1), assertion failures, break). These lines may be marked as uncovered even when executed, due to JaCoCo's limitations in tracking code execution after abrupt control flow interruptions. (See tracking issue [#5506](https://github.com/oppia/oppia-android/issues/5506)) + +6. **Uncovered Last Curly Brace in Kotlin:** The last curly brace of some Kotlin functions may be reported as uncovered, even when the function is fully executed during tests. This issue requires further investigation to determine if it's due to incomplete test execution or dead code generated by the Kotlin compiler. (See tracking issue [#5523](https://github.com/oppia/oppia-android/issues/5523)) \ No newline at end of file diff --git a/wiki/Writing-tests-with-good-behavioral-coverage.md b/wiki/Writing-tests-with-good-behavioral-coverage.md index 7a0aa5caebc..db50cfa83a6 100644 --- a/wiki/Writing-tests-with-good-behavioral-coverage.md +++ b/wiki/Writing-tests-with-good-behavioral-coverage.md @@ -1017,7 +1017,7 @@ Note: For more information on how to utilize the code coverage analysis tool, pl ## Testing a Single Outcome in Multiple Ways -When testing a single outcome like a successful withdrawal, you can use multiple approaches to verify the if the balance is updated correctly. Here are different ways to ensure the single outcome of withdrawal was processed correctly, each following a distinct approach. +When testing a single outcome, such as a successful withdrawal, you can use multiple approaches to verify if the balance is updated correctly. Here are different ways to ensure the single outcome of withdrawal was processed correctly, each following a distinct approach. **a. To verify correctness of output:** diff --git a/wiki/_Sidebar.md b/wiki/_Sidebar.md index ea5a254f2c6..c1f9834bc49 100644 --- a/wiki/_Sidebar.md +++ b/wiki/_Sidebar.md @@ -25,7 +25,7 @@ * Testing * [Oppia Android Testing](https://github.com/oppia/oppia-android/wiki/Oppia-Android-Testing) * [End to End Testing Guide](https://github.com/oppia/oppia-android/wiki/End-to-End-Testing-Guide) - * [Oppia Android Code Coverage](https://github.com/oppia/oppia-android-workflow/wiki/Oppia-Android-Code-Coverage) + * [Oppia Android Code Coverage](https://github.com/oppia/oppia-android/wiki/Oppia-Android-Code-Coverage) * [Writing Tests with Good Behavioral Coverage](https://github.com/oppia/oppia-android/wiki/Writing-Tests-With-Good-Behavioral-Coverage) * [Developing Skills](https://github.com/oppia/oppia-android/wiki/Developing-skills) * [Frequent Errors and Solutions](https://github.com/oppia/oppia-android/wiki/Frequent-Errors-and-Solutions)