diff --git a/scripts/src/java/org/oppia/android/scripts/common/BazelClient.kt b/scripts/src/java/org/oppia/android/scripts/common/BazelClient.kt index 9aede2fd16f..62ce53bfb82 100644 --- a/scripts/src/java/org/oppia/android/scripts/common/BazelClient.kt +++ b/scripts/src/java/org/oppia/android/scripts/common/BazelClient.kt @@ -130,6 +130,40 @@ class BazelClient(private val rootDirectory: File, private val commandExecutor: return correctedTargets } + /** + * Runs code coverage for the specified Bazel test target. + * + * Null return typically occurs when the coverage command fails to generate the 'coverage.dat' file + * This can happen due to: Test failures or misconfigurations that prevent the coverage data + * from being generated properly. + * + * @param bazelTestTarget Bazel test target for which code coverage will be run + * @return the generated coverage data as a list of strings + * or null if the coverage data file could not be parsed + */ + fun runCoverageForTestTarget(bazelTestTarget: String): List? { + val coverageCommandOutputLines = executeBazelCommand( + "coverage", + bazelTestTarget + ) + return parseCoverageDataFilePath(coverageCommandOutputLines)?.let { path -> + File(path).readLines() + } + } + + private fun parseCoverageDataFilePath(coverageCommandOutputLines: List): String? { + val regex = ".*coverage\\.dat$".toRegex() + for (line in coverageCommandOutputLines) { + val match = regex.find(line) + val extractedPath = match?.value?.substringAfterLast(",")?.trim() + if (extractedPath != null) { + println("Parsed Coverage Data File: $extractedPath") + return extractedPath + } + } + return null + } + /** * Returns the results of a query command with a potentially large list of [values] that will be * split up into multiple commands to avoid overflow the system's maximum argument limit. diff --git a/scripts/src/java/org/oppia/android/scripts/coverage/BUILD.bazel b/scripts/src/java/org/oppia/android/scripts/coverage/BUILD.bazel new file mode 100644 index 00000000000..4c7ab41a5fa --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/coverage/BUILD.bazel @@ -0,0 +1,17 @@ +""" +Libraries corresponding to developer scripts that obtain coverage data for test targets. +""" + +load("@io_bazel_rules_kotlin//kotlin:kotlin.bzl", "kt_jvm_library") + +kt_jvm_library( + name = "coverage_runner", + testonly = True, + srcs = [ + "CoverageRunner.kt", + ], + visibility = ["//scripts:oppia_script_binary_visibility"], + deps = [ + "//scripts/src/java/org/oppia/android/scripts/common:bazel_client", + ], +) diff --git a/scripts/src/java/org/oppia/android/scripts/coverage/CoverageRunner.kt b/scripts/src/java/org/oppia/android/scripts/coverage/CoverageRunner.kt new file mode 100644 index 00000000000..e4701cd2da3 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/coverage/CoverageRunner.kt @@ -0,0 +1,44 @@ +package org.oppia.android.scripts.coverage + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import org.oppia.android.scripts.common.BazelClient +import org.oppia.android.scripts.common.CommandExecutor +import org.oppia.android.scripts.common.ScriptBackgroundCoroutineDispatcher +import java.io.File + +/** + * Class responsible for running coverage analysis asynchronously. + * + * @param repoRoot the root directory of the repository + * @param scriptBgDispatcher the [ScriptBackgroundCoroutineDispatcher] to be used for running the coverage command + * @param commandExecutor executes the specified command in the specified working directory + */ +class CoverageRunner( + private val repoRoot: File, + private val scriptBgDispatcher: ScriptBackgroundCoroutineDispatcher, + private val commandExecutor: CommandExecutor +) { + private val bazelClient by lazy { BazelClient(repoRoot, commandExecutor) } + + /** + * Runs coverage analysis asynchronously for the Bazel test target. + * + * @param bazelTestTarget Bazel test target to analyze coverage + * @return a deferred value that contains the coverage data + */ + fun runWithCoverageAsync( + bazelTestTarget: String + ): Deferred?> { + return CoroutineScope(scriptBgDispatcher).async { + retrieveCoverageResult(bazelTestTarget) + } + } + + private fun retrieveCoverageResult( + bazelTestTarget: String + ): List? { + return bazelClient.runCoverageForTestTarget(bazelTestTarget) + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/testing/TestBazelWorkspace.kt b/scripts/src/java/org/oppia/android/scripts/testing/TestBazelWorkspace.kt index 7c57cf0d10c..71ee2eb542a 100644 --- a/scripts/src/java/org/oppia/android/scripts/testing/TestBazelWorkspace.kt +++ b/scripts/src/java/org/oppia/android/scripts/testing/TestBazelWorkspace.kt @@ -50,6 +50,141 @@ class TestBazelWorkspace(private val temporaryRootFolder: TemporaryFolder) { assertThat(bazelRcFile.exists()).isTrue() } + /** + * Adds a source file and test file with the specified name and content, + * and updates the corresponding build configuration. + * + * @param filename the name of the source file (without the .kt extension) + * @param sourceContent the content of the source file + * @param testContent the content of the test file + * @param subpackage the subpackage under which the source and test files should be added + */ + fun addSourceAndTestFileWithContent( + filename: String, + sourceContent: String, + testContent: String, + subpackage: String + ) { + val sourceSubpackage = "$subpackage/main/java/com/example" + addSourceContentAndBuildFile( + filename, + sourceContent, + sourceSubpackage + ) + + val testSubpackage = "$subpackage/test/java/com/example" + val testFileName = "${filename}Test" + addTestContentAndBuildFile( + filename, + testFileName, + testContent, + sourceSubpackage, + testSubpackage + ) + } + + /** + * Adds a source file with the specified name and content to the specified subpackage, + * and updates the corresponding build configuration. + * + * @param filename the name of the source file (without the .kt extension) + * @param sourceContent the content of the source file + * @param sourceSubpackage the subpackage under which the source file should be added + * @return the target name of the added source file + */ + fun addSourceContentAndBuildFile( + filename: String, + sourceContent: String, + sourceSubpackage: String + ) { + initEmptyWorkspace() + ensureWorkspaceIsConfiguredForKotlin() + setUpWorkspaceForRulesJvmExternal( + listOf("junit:junit:4.12") + ) + + // Create the source subpackage directory if it doesn't exist + if (!File(temporaryRootFolder.root, sourceSubpackage.replace(".", "/")).exists()) { + temporaryRootFolder.newFolder(*(sourceSubpackage.split(".")).toTypedArray()) + } + + // Create the source file + val sourceFile = temporaryRootFolder.newFile( + "${sourceSubpackage.replace(".", "/")}/$filename.kt" + ) + sourceFile.writeText(sourceContent) + + // Create or update the BUILD file for the source file + val buildFileRelativePath = "${sourceSubpackage.replace(".", "/")}/BUILD.bazel" + val buildFile = File(temporaryRootFolder.root, buildFileRelativePath) + if (!buildFile.exists()) { + temporaryRootFolder.newFile(buildFileRelativePath) + } + prepareBuildFileForLibraries(buildFile) + + buildFile.appendText( + """ + kt_jvm_library( + name = "${filename.lowercase()}", + srcs = ["$filename.kt"], + visibility = ["//visibility:public"] + ) + """.trimIndent() + "\n" + ) + } + + /** + * Adds a test file with the specified name and content to the specified subpackage, + * and updates the corresponding build configuration. + * + * @param filename the name of the source file (without the .kt extension) + * @param testName the name of the test file (without the .kt extension) + * @param testContent the content of the test file + * @param testSubpackage the subpackage for the test file + */ + fun addTestContentAndBuildFile( + filename: String, + testName: String, + testContent: String, + sourceSubpackage: String, + testSubpackage: String + ) { + initEmptyWorkspace() + + // Create the test subpackage directory for the test file if it doesn't exist + if (!File(temporaryRootFolder.root, testSubpackage.replace(".", "/")).exists()) { + temporaryRootFolder.newFolder(*(testSubpackage.split(".")).toTypedArray()) + } + + // Create the test file + val testFile = temporaryRootFolder.newFile("${testSubpackage.replace(".", "/")}/$testName.kt") + testFile.writeText(testContent) + + // Create or update the BUILD file for the test file + val testBuildFileRelativePath = "${testSubpackage.replace(".", "/")}/BUILD.bazel" + val testBuildFile = File(temporaryRootFolder.root, testBuildFileRelativePath) + if (!testBuildFile.exists()) { + temporaryRootFolder.newFile(testBuildFileRelativePath) + } + prepareBuildFileForTests(testBuildFile) + + // Add the test file to the BUILD file with appropriate dependencies + testBuildFile.appendText( + """ + kt_jvm_test( + name = "test", + srcs = ["$testName.kt"], + deps = [ + "//$sourceSubpackage:${filename.lowercase()}", + "@maven//:junit_junit", + ], + visibility = ["//visibility:public"], + test_class = "com.example.$testName", + ) + """.trimIndent() + "\n" + ) + } + /** * Generates and adds a new kt_jvm_test target with the target name [testName] and test file * [testFile]. This can be used to add multiple tests to the same build file, and will diff --git a/scripts/src/javatests/org/oppia/android/scripts/common/BazelClientTest.kt b/scripts/src/javatests/org/oppia/android/scripts/common/BazelClientTest.kt index 3f451d354b8..075feeab3fe 100644 --- a/scripts/src/javatests/org/oppia/android/scripts/common/BazelClientTest.kt +++ b/scripts/src/javatests/org/oppia/android/scripts/common/BazelClientTest.kt @@ -22,8 +22,8 @@ import java.util.concurrent.TimeUnit * Note that this test executes real commands on the local filesystem & requires Bazel in the local * environment. */ -// Same parameter value: helpers reduce test context, even if they are used by 1 test. -// Function name: test names are conventionally named with underscores. +// Same parameter value: helpers reduce test context, even if they are used by 1 test +// Function name: test names are conventionally named with underscores @Suppress("SameParameterValue", "FunctionName") class BazelClientTest { @field:[Rule JvmField] val tempFolder = TemporaryFolder() @@ -379,6 +379,95 @@ class BazelClientTest { assertThat(thirdPartyDependenciesList).doesNotContain("@maven//:androidx_annotation_annotation") } + @Test + fun testRunCodeCoverage_forSampleTestTarget_returnsCoverageResult() { + val bazelClient = BazelClient(tempFolder.root, longCommandExecutor) + testBazelWorkspace.initEmptyWorkspace() + + val sourceContent = + """ + package com.example + + class TwoSum { + + companion object { + fun sumNumbers(a: Int, b: Int): Any { + return if (a == 0 && b == 0) { + "Both numbers are zero" + } else { + a + b + } + } + } + } + """.trimIndent() + + val testContent = + """ + package com.example + + import org.junit.Assert.assertEquals + import org.junit.Test + + class TwoSumTest { + + @Test + fun testSumNumbers() { + assertEquals(TwoSum.sumNumbers(0, 1), 1) + assertEquals(TwoSum.sumNumbers(3, 4), 7) + assertEquals(TwoSum.sumNumbers(0, 0), "Both numbers are zero") + } + } + """.trimIndent() + + testBazelWorkspace.addSourceAndTestFileWithContent( + filename = "TwoSum", + sourceContent = sourceContent, + testContent = testContent, + subpackage = "coverage" + ) + + val result = bazelClient.runCoverageForTestTarget("//coverage/test/java/com/example:test") + val expectedResult = listOf( + "SF:coverage/main/java/com/example/TwoSum.kt", + "FN:7,com/example/TwoSum${'$'}Companion::sumNumbers (II)Ljava/lang/Object;", + "FN:3,com/example/TwoSum:: ()V", + "FNDA:1,com/example/TwoSum${'$'}Companion::sumNumbers (II)Ljava/lang/Object;", + "FNDA:0,com/example/TwoSum:: ()V", + "FNF:2", + "FNH:1", + "BRDA:7,0,0,1", + "BRDA:7,0,1,1", + "BRDA:7,0,2,1", + "BRDA:7,0,3,1", + "BRF:4", + "BRH:4", + "DA:3,0", + "DA:7,1", + "DA:8,1", + "DA:10,1", + "LH:3", + "LF:4", + "end_of_record" + ) + + assertThat(result).isEqualTo(expectedResult) + } + + @Test + fun testRunCodeCoverage_forNonTestTarget_fails() { + val bazelClient = BazelClient(tempFolder.root, longCommandExecutor) + testBazelWorkspace.initEmptyWorkspace() + + val exception = assertThrows() { + bazelClient.runCoverageForTestTarget("//coverage/test/java/com/example:test") + } + + // Verify that the underlying Bazel command failed since the test target was not available. + assertThat(exception).hasMessageThat().contains("Expected non-zero exit code") + assertThat(exception).hasMessageThat().contains("no such package") + } + private fun fakeCommandExecutorWithResult(singleLine: String) { // Fake a Bazel command's results to return jumbled results. This has been observed to happen // sometimes in CI, but doesn't have a known cause. The utility is meant to de-jumble these in diff --git a/scripts/src/javatests/org/oppia/android/scripts/coverage/BUILD.bazel b/scripts/src/javatests/org/oppia/android/scripts/coverage/BUILD.bazel new file mode 100644 index 00000000000..f2e5c80564b --- /dev/null +++ b/scripts/src/javatests/org/oppia/android/scripts/coverage/BUILD.bazel @@ -0,0 +1,17 @@ +""" +Tests corresponding to developer scripts that help with obtaining coverage data for test targets. +""" + +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_test") + +kt_jvm_test( + name = "CoverageRunnerTest", + srcs = ["CoverageRunnerTest.kt"], + deps = [ + "//scripts/src/java/org/oppia/android/scripts/coverage:coverage_runner", + "//scripts/src/java/org/oppia/android/scripts/testing:test_bazel_workspace", + "//testing:assertion_helpers", + "//third_party:com_google_truth_truth", + "//third_party:org_jetbrains_kotlin_kotlin-test-junit", + ], +) diff --git a/scripts/src/javatests/org/oppia/android/scripts/coverage/CoverageRunnerTest.kt b/scripts/src/javatests/org/oppia/android/scripts/coverage/CoverageRunnerTest.kt new file mode 100644 index 00000000000..bae58d98feb --- /dev/null +++ b/scripts/src/javatests/org/oppia/android/scripts/coverage/CoverageRunnerTest.kt @@ -0,0 +1,147 @@ +package org.oppia.android.scripts.coverage + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.runBlocking +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.oppia.android.scripts.common.CommandExecutorImpl +import org.oppia.android.scripts.common.ScriptBackgroundCoroutineDispatcher +import org.oppia.android.scripts.testing.TestBazelWorkspace +import org.oppia.android.testing.assertThrows +import java.util.concurrent.TimeUnit + +/** Tests for [CoverageRunner]. */ +class CoverageRunnerTest { + @field:[Rule JvmField] val tempFolder = TemporaryFolder() + + private val scriptBgDispatcher by lazy { ScriptBackgroundCoroutineDispatcher() } + private val longCommandExecutor by lazy { initializeCommandExecutorWithLongProcessWaitTime() } + + private lateinit var coverageRunner: CoverageRunner + private lateinit var testBazelWorkspace: TestBazelWorkspace + private lateinit var bazelTestTarget: String + + @Before + fun setUp() { + coverageRunner = CoverageRunner(tempFolder.root, scriptBgDispatcher, longCommandExecutor) + bazelTestTarget = "//:testTarget" + testBazelWorkspace = TestBazelWorkspace(tempFolder) + } + + @After + fun tearDown() { + scriptBgDispatcher.close() + } + + @Test + fun testRunCoverage_emptyDirectory_throwsException() { + val exception = assertThrows() { + runBlocking { + coverageRunner.runWithCoverageAsync(bazelTestTarget).await() + } + } + + assertThat(exception).hasMessageThat().contains("not invoked from within a workspace") + } + + @Test + fun testRunCoverage_invalidTestTarget_throwsException() { + testBazelWorkspace.initEmptyWorkspace() + + val exception = assertThrows() { + runBlocking { + coverageRunner.runWithCoverageAsync(bazelTestTarget).await() + } + } + + assertThat(exception).hasMessageThat().contains("Expected non-zero exit code") + assertThat(exception).hasMessageThat().contains("no such package") + } + + @Test + fun testRunCoverage_validSampleTestTarget_returnsCoverageData() { + testBazelWorkspace.initEmptyWorkspace() + + val sourceContent = + """ + package com.example + + class TwoSum { + + companion object { + fun sumNumbers(a: Int, b: Int): Any { + return if (a ==0 && b == 0) { + "Both numbers are zero" + } else { + a + b + } + } + } + } + """.trimIndent() + + val testContent = + """ + package com.example + + import org.junit.Assert.assertEquals + import org.junit.Test + + class TwoSumTest { + + @Test + fun testSumNumbers() { + assertEquals(TwoSum.sumNumbers(0, 1), 1) + assertEquals(TwoSum.sumNumbers(3, 4), 7) + assertEquals(TwoSum.sumNumbers(0, 0), "Both numbers are zero") + } + } + """.trimIndent() + + testBazelWorkspace.addSourceAndTestFileWithContent( + filename = "TwoSum", + sourceContent = sourceContent, + testContent = testContent, + subpackage = "coverage" + ) + + val result = runBlocking { + coverageRunner.runWithCoverageAsync( + "//coverage/test/java/com/example:test" + ).await() + } + val expectedResult = listOf( + "SF:coverage/main/java/com/example/TwoSum.kt", + "FN:7,com/example/TwoSum${'$'}Companion::sumNumbers (II)Ljava/lang/Object;", + "FN:3,com/example/TwoSum:: ()V", + "FNDA:1,com/example/TwoSum${'$'}Companion::sumNumbers (II)Ljava/lang/Object;", + "FNDA:0,com/example/TwoSum:: ()V", + "FNF:2", + "FNH:1", + "BRDA:7,0,0,1", + "BRDA:7,0,1,1", + "BRDA:7,0,2,1", + "BRDA:7,0,3,1", + "BRF:4", + "BRH:4", + "DA:3,0", + "DA:7,1", + "DA:8,1", + "DA:10,1", + "LH:3", + "LF:4", + "end_of_record" + ) + + assertThat(result).isEqualTo(expectedResult) + } + + private fun initializeCommandExecutorWithLongProcessWaitTime(): CommandExecutorImpl { + return CommandExecutorImpl( + scriptBgDispatcher, processTimeout = 5, processTimeoutUnit = TimeUnit.MINUTES + ) + } +} diff --git a/scripts/src/javatests/org/oppia/android/scripts/testing/TestBazelWorkspaceTest.kt b/scripts/src/javatests/org/oppia/android/scripts/testing/TestBazelWorkspaceTest.kt index 2ea2f42b1bf..9d6a33378d6 100644 --- a/scripts/src/javatests/org/oppia/android/scripts/testing/TestBazelWorkspaceTest.kt +++ b/scripts/src/javatests/org/oppia/android/scripts/testing/TestBazelWorkspaceTest.kt @@ -254,6 +254,171 @@ class TestBazelWorkspaceTest { assertThat(buildFile.length()).isNotEqualTo(originalLength) } + @Test + fun testAddSourceAndTestFileWithContent_createsSourceAndTestFiles() { + val testBazelWorkspace = TestBazelWorkspace(tempFolder) + val sourceContent = + """ + fun main() { + println("Hello, World!") + } + """ + + val testContent = + """ + import org.junit.Test + import kotlin.test.assertEquals + + class MainTest { + + @Test + fun testMain() { + assertEquals(1, 1) + } + } + """ + + testBazelWorkspace.addSourceAndTestFileWithContent( + "Main", + sourceContent, + testContent, + "coverage" + ) + + val sourceFile = File(tempFolder.root, "coverage/main/java/com/example/Main.kt") + val testFile = File(tempFolder.root, "coverage/test/java/com/example/MainTest.kt") + + assertThat(sourceFile.exists()).isTrue() + assertThat(sourceFile.readText()).isEqualTo(sourceContent) + + assertThat(testFile.exists()).isTrue() + assertThat(testFile.readText()).isEqualTo(testContent) + } + + @Test + fun testAddSourceAndTestFileWithContent_updatesBuildFiles() { + val testBazelWorkspace = TestBazelWorkspace(tempFolder) + val sourceContent = "fun main() { println(\"Hello, World!\") }" + val testContent = + """ + import org.junit.Test + import kotlin.test.assertEquals + + class MainTest { + @Test + fun testMain() { + assertEquals(1, 1) + } + } + """.trimIndent() + + testBazelWorkspace.addSourceAndTestFileWithContent( + "Main", + sourceContent, + testContent, + "coverage" + ) + + val sourceBuildFile = File(tempFolder.root, "coverage/main/java/com/example/BUILD.bazel") + val testBuildFile = File(tempFolder.root, "coverage/test/java/com/example/BUILD.bazel") + + assertThat(sourceBuildFile.exists()).isTrue() + assertThat(sourceBuildFile.readText()).contains( + """ + kt_jvm_library( + name = "main", + srcs = ["Main.kt"], + visibility = ["//visibility:public"] + ) + """.trimIndent() + ) + + assertThat(testBuildFile.exists()).isTrue() + assertThat(testBuildFile.readText()).contains( + """ + load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_test") + kt_jvm_test( + name = "test", + srcs = ["MainTest.kt"], + deps = [ + "//coverage/main/java/com/example:main", + "@maven//:junit_junit", + ], + visibility = ["//visibility:public"], + test_class = "com.example.MainTest", + ) + """.trimIndent() + ) + } + + @Test + fun testAddSourceContentAndBuildFile_createsSourceFileAndBuildFile() { + val testBazelWorkspace = TestBazelWorkspace(tempFolder) + val sourceContent = "fun main() { println(\"Hello, World!\") }" + + testBazelWorkspace.addSourceContentAndBuildFile( + "Main", + sourceContent, + "coverage/main/java/com/example" + ) + + val sourceFile = File(tempFolder.root, "coverage/main/java/com/example/Main.kt") + val buildFile = File(tempFolder.root, "coverage/main/java/com/example/BUILD.bazel") + + assertThat(sourceFile.exists()).isTrue() + assertThat(sourceFile.readText()).isEqualTo(sourceContent.trimIndent()) + + assertThat(buildFile.exists()).isTrue() + assertThat(buildFile.readText()).contains( + """ + kt_jvm_library( + name = "main", + srcs = ["Main.kt"], + visibility = ["//visibility:public"] + ) + """.trimIndent() + ) + } + + @Test + fun testAddTestContentAndBuildFile_createsTestFileAndBuildFile() { + val testBazelWorkspace = TestBazelWorkspace(tempFolder) + val testContent = "import org.junit.Test" + + "\nimport kotlin.test.assertEquals\n\nclass MainTest {" + + "\n@Test\nfun testMain() {\nassertEquals(1, 1)\n}\n}" + + testBazelWorkspace.addTestContentAndBuildFile( + "Main", + "MainTest", + testContent, + "coverage/main/java/com/example", + "coverage/test/java/com/example" + ) + + val testFile = File(tempFolder.root, "coverage/test/java/com/example/MainTest.kt") + val buildFile = File(tempFolder.root, "coverage/test/java/com/example/BUILD.bazel") + + assertThat(testFile.exists()).isTrue() + assertThat(testFile.readText()).isEqualTo(testContent.trimIndent()) + + assertThat(buildFile.exists()).isTrue() + assertThat(buildFile.readText()).contains( + """ + load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_test") + kt_jvm_test( + name = "test", + srcs = ["MainTest.kt"], + deps = [ + "//coverage/main/java/com/example:main", + "@maven//:junit_junit", + ], + visibility = ["//visibility:public"], + test_class = "com.example.MainTest", + ) + """.trimIndent() + ) + } + @Test fun testAddTestToBuildFile_reusedTestName_throwsException() { val testBazelWorkspace = TestBazelWorkspace(tempFolder)