From e39d67402f86f9ebd1c29b1d2cbdca3acb414b80 Mon Sep 17 00:00:00 2001 From: Zac Sweers Date: Thu, 15 Jun 2023 16:36:24 -0400 Subject: [PATCH] Upstream processed exec implementation (#20) * Switch to Path for projectDirOption * Initial port of processed exec * Make issues fully configurable * Standardize envs a bit * Comment a bug * Make JSON more canonical * Add a bugsnag artifact upload failures * Add build scan server to config + test * Fix exit code not falling through * Use duration in message * Spotless --- api/kotlin-cli-util.api | 33 ++++ build.gradle.kts | 10 ++ gradle/libs.versions.toml | 7 + src/main/kotlin/slack/cli/CliktExtensions.kt | 10 +- src/main/kotlin/slack/cli/exec/Issue.kt | 65 +++++++ src/main/kotlin/slack/cli/exec/KnownIssues.kt | 97 +++++++++++ .../slack/cli/exec/OkHttpSyncHttpDelivery.kt | 88 ++++++++++ .../kotlin/slack/cli/exec/ProcessedExecCli.kt | 153 +++++++++++++++++ .../slack/cli/exec/ProcessedExecConfig.kt | 38 +++++ .../kotlin/slack/cli/exec/ProcessingUtil.kt | 27 +++ .../kotlin/slack/cli/exec/ResultProcessor.kt | 159 ++++++++++++++++++ src/main/kotlin/slack/cli/exec/RetrySignal.kt | 41 +++++ .../slack/cli/util/DurationJsonAdapter.kt | 33 ++++ .../slack/cli/exec/ProcessedExecConfigTest.kt | 66 ++++++++ .../slack/cli/exec/ResultProcessorTest.kt | 136 +++++++++++++++ 15 files changed, 958 insertions(+), 5 deletions(-) create mode 100644 src/main/kotlin/slack/cli/exec/Issue.kt create mode 100644 src/main/kotlin/slack/cli/exec/KnownIssues.kt create mode 100644 src/main/kotlin/slack/cli/exec/OkHttpSyncHttpDelivery.kt create mode 100644 src/main/kotlin/slack/cli/exec/ProcessedExecCli.kt create mode 100644 src/main/kotlin/slack/cli/exec/ProcessedExecConfig.kt create mode 100644 src/main/kotlin/slack/cli/exec/ProcessingUtil.kt create mode 100644 src/main/kotlin/slack/cli/exec/ResultProcessor.kt create mode 100644 src/main/kotlin/slack/cli/exec/RetrySignal.kt create mode 100644 src/main/kotlin/slack/cli/util/DurationJsonAdapter.kt create mode 100644 src/test/kotlin/slack/cli/exec/ProcessedExecConfigTest.kt create mode 100644 src/test/kotlin/slack/cli/exec/ResultProcessorTest.kt diff --git a/api/kotlin-cli-util.api b/api/kotlin-cli-util.api index 85cd331..29c6082 100644 --- a/api/kotlin-cli-util.api +++ b/api/kotlin-cli-util.api @@ -36,3 +36,36 @@ public final class slack/cli/Toml { public final fun parseVersion (Ljava/io/File;)Ljava/util/Map; } +public final class slack/cli/exec/IssueJsonAdapter : com/squareup/moshi/JsonAdapter { + public fun (Lcom/squareup/moshi/Moshi;)V + public fun fromJson (Lcom/squareup/moshi/JsonReader;)Ljava/lang/Object; + public fun toJson (Lcom/squareup/moshi/JsonWriter;Ljava/lang/Object;)V + public fun toString ()Ljava/lang/String; +} + +public final class slack/cli/exec/ProcessedExecCli : com/github/ajalt/clikt/core/CliktCommand { + public fun ()V + public fun run ()V +} + +public final class slack/cli/exec/ProcessedExecConfigJsonAdapter : com/squareup/moshi/JsonAdapter { + public fun (Lcom/squareup/moshi/Moshi;)V + public fun fromJson (Lcom/squareup/moshi/JsonReader;)Ljava/lang/Object; + public fun toJson (Lcom/squareup/moshi/JsonWriter;Ljava/lang/Object;)V + public fun toString ()Ljava/lang/String; +} + +public final class slack/cli/exec/RetrySignalJsonAdapter : com/squareup/moshi/JsonAdapter { + public fun (Lcom/squareup/moshi/Moshi;)V + public fun fromJson (Lcom/squareup/moshi/JsonReader;)Ljava/lang/Object; + public fun toJson (Lcom/squareup/moshi/JsonWriter;Ljava/lang/Object;)V + public fun toString ()Ljava/lang/String; +} + +public final class slack/cli/exec/RetrySignal_RetryDelayedJsonAdapter : com/squareup/moshi/JsonAdapter { + public fun (Lcom/squareup/moshi/Moshi;)V + public fun fromJson (Lcom/squareup/moshi/JsonReader;)Ljava/lang/Object; + public fun toJson (Lcom/squareup/moshi/JsonWriter;Ljava/lang/Object;)V + public fun toString ()Ljava/lang/String; +} + diff --git a/build.gradle.kts b/build.gradle.kts index e70a9d0..849ed11 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,6 +26,7 @@ plugins { alias(libs.plugins.mavenPublish) alias(libs.plugins.spotless) alias(libs.plugins.binaryCompatibilityValidator) + alias(libs.plugins.moshix) } spotless { @@ -77,10 +78,19 @@ tasks.withType().configureEach { kotlin { explicitApi() } +moshi { + enableSealed.set(true) + generateProguardRules.set(false) +} + dependencies { api(libs.clikt) implementation(libs.kotlinShell) implementation(libs.okio) + implementation(libs.okhttp) + implementation(libs.bugsnag) + implementation(libs.moshi) + implementation(libs.kotlin.reflect) testImplementation(libs.junit) testImplementation(libs.truth) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5854028..6be8fde 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,19 +2,26 @@ kotlin = "1.8.22" ktfmt = "0.43" jvmTarget = "17" +moshix = "0.22.1" +moshi = "1.15.0" [plugins] detekt = { id = "io.gitlab.arturbosch.detekt", version = "1.23.0" } dokka = { id = "org.jetbrains.dokka", version = "1.8.20" } lint = { id = "com.android.lint", version = "8.0.2" } mavenPublish = { id = "com.vanniktech.maven.publish", version = "0.25.2" } +moshix = { id = "dev.zacsweers.moshix", version.ref = "moshix" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } spotless = { id = "com.diffplug.spotless", version = "6.19.0" } binaryCompatibilityValidator = { id = "org.jetbrains.kotlinx.binary-compatibility-validator", version = "0.13.2" } [libraries] +bugsnag = "com.bugsnag:bugsnag:3.6.4" clikt = "com.github.ajalt.clikt:clikt:3.5.2" kotlinShell = "eu.jrie.jetbrains:kotlin-shell-core:0.2.1" +kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" } +moshi = { module = "com.squareup.moshi:moshi", version.ref = "moshi" } +okhttp = "com.squareup.okhttp3:okhttp:4.11.0" okio = "com.squareup.okio:okio:3.3.0" junit = "junit:junit:4.13.2" truth = "com.google.truth:truth:1.1.4" \ No newline at end of file diff --git a/src/main/kotlin/slack/cli/CliktExtensions.kt b/src/main/kotlin/slack/cli/CliktExtensions.kt index d8e3c20..0ae88e7 100644 --- a/src/main/kotlin/slack/cli/CliktExtensions.kt +++ b/src/main/kotlin/slack/cli/CliktExtensions.kt @@ -27,8 +27,8 @@ import com.github.ajalt.clikt.parameters.options.OptionDelegate import com.github.ajalt.clikt.parameters.options.defaultLazy import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.types.file -import java.io.File +import com.github.ajalt.clikt.parameters.types.path +import java.nio.file.Path import java.nio.file.Paths /** A dry run option for [clikt commands][CliktCommand]. */ @@ -41,7 +41,7 @@ public fun CliktCommand.dryRunOption( public fun CliktCommand.projectDirOption( vararg names: String = arrayOf("--project-dir"), help: String = "The project directory. Defaults to the current working directory." -): OptionDelegate = - option(names = names, help = help).file(mustExist = true, canBeFile = false).defaultLazy { - Paths.get("").toFile().canonicalFile +): OptionDelegate = + option(names = names, help = help).path(mustExist = true, canBeFile = false).defaultLazy { + Paths.get("").toAbsolutePath() } diff --git a/src/main/kotlin/slack/cli/exec/Issue.kt b/src/main/kotlin/slack/cli/exec/Issue.kt new file mode 100644 index 0000000..d8b4036 --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/Issue.kt @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +/** + * An issue that can be reported to Bugsnag. + * + * @property message the message shown in the bugsnag report message. Should be human-readable. + * @property logMessage the message shown in the CI log when [matchingText] is found. Should be + * human-readable. + * @property matchingText the matching text to look for in the log. + * @property groupingHash grouping hash for reporting to bugsnag. This should usually be unique, but + * can also be reused across issues that are part of the same general issue. + */ +@JsonClass(generateAdapter = true) +internal data class Issue( + val message: String, + @Json(name = "log_message") val logMessage: String, + @Json(name = "matching_text") val matchingText: String, + @Json(name = "grouping_hash") val groupingHash: String, + @Json(name = "retry_signal") val retrySignal: RetrySignal +) { + + private fun List.checkContains(errorText: String): Boolean { + return any { it.contains(errorText, ignoreCase = true) } + } + + /** Checks the log for this issue and returns a [RetrySignal] if it should be retried. */ + fun check(lines: List, log: (String) -> Unit): RetrySignal { + return if (lines.checkContains(matchingText)) { + log(logMessage) + retrySignal + } else { + RetrySignal.Unknown + } + } +} + +/** + * Base class for an issue that can be reported to Bugsnag. This is a [Throwable] for BugSnag + * purposes but doesn't fill in a stacktrace. + */ +internal class IssueThrowable(issue: Issue) : Throwable(issue.message) { + + override fun fillInStackTrace(): Throwable { + // Do nothing, the stacktrace isn't relevant for these! + return this + } +} diff --git a/src/main/kotlin/slack/cli/exec/KnownIssues.kt b/src/main/kotlin/slack/cli/exec/KnownIssues.kt new file mode 100644 index 0000000..d8cda4f --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/KnownIssues.kt @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import kotlin.time.Duration.Companion.minutes + +private const val OOM_GROUPING_HASH = "oom" + +/** A set of known issues. */ +@Suppress("unused") // We look these up reflectively at runtime +internal object KnownIssues { + // A simple fake checker for testing this script + val fakeFailure = + Issue( + message = "Fake failure", + logMessage = "Detected fake failure. Beep boop.", + matchingText = "FAKE FAILURE NOT REAL", + groupingHash = "fake-failure", + retrySignal = RetrySignal.Ack + ) + + val ftlRateLimit = + Issue( + message = "FTL rate limit", + matchingText = "429 Too Many Requests", + logMessage = "Detected FTL rate limit. Retrying in 1 minute.", + groupingHash = "ftl-rate-limit", + retrySignal = RetrySignal.RetryDelayed(1.minutes) + ) + + val oom = + Issue( + message = "Generic OOM", + matchingText = "Java heap space", + logMessage = "Detected OOM. Retrying immediately.", + groupingHash = OOM_GROUPING_HASH, + retrySignal = RetrySignal.RetryImmediately + ) + + val ftlInfrastructureFailure = + Issue( + message = "Inconclusive FTL infrastructure failure", + matchingText = "Infrastructure failure", + logMessage = "Detected inconclusive FTL infrastructure failure. Retrying immediately.", + groupingHash = "ftl-infrastructure-failure", + retrySignal = RetrySignal.RetryImmediately + ) + + val flankTimeout = + Issue( + message = "Flank timeout", + groupingHash = "flank-timeout", + matchingText = "Canceling flank due to timeout", + logMessage = "Detected a flank timeout. Retrying immediately.", + retrySignal = RetrySignal.RetryImmediately + ) + + val r8Oom = + Issue( + message = "R8 OOM", + matchingText = "Out of space in CodeCache", + logMessage = "Detected a OOM in R8. Retrying immediately.", + groupingHash = OOM_GROUPING_HASH, + retrySignal = RetrySignal.RetryImmediately + ) + + val oomKilledByKernel = + Issue( + message = "OOM killed by kernel", + groupingHash = OOM_GROUPING_HASH, + matchingText = "Gradle build daemon disappeared unexpectedly", + logMessage = "Detected a OOM that was killed by the kernel. Retrying immediately.", + retrySignal = RetrySignal.RetryImmediately + ) + + val bugsnagUploadFailed = + Issue( + message = "Bugsnag artifact upload failure", + groupingHash = "bugsnag-upload-failure", + matchingText = "Bugsnag request failed to complete", + logMessage = "Detected bugsnag failed to upload. Retrying immediately.", + retrySignal = RetrySignal.RetryImmediately + ) +} diff --git a/src/main/kotlin/slack/cli/exec/OkHttpSyncHttpDelivery.kt b/src/main/kotlin/slack/cli/exec/OkHttpSyncHttpDelivery.kt new file mode 100644 index 0000000..fcd71db --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/OkHttpSyncHttpDelivery.kt @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.bugsnag.delivery.HttpDelivery +import com.bugsnag.serialization.Serializer +import java.io.IOException +import java.net.Proxy +import java.util.concurrent.TimeUnit +import okhttp3.Authenticator +import okhttp3.Headers.Companion.toHeaders +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import okio.BufferedSink +import org.slf4j.LoggerFactory + +/** An [OkHttpClient]-based implementation of Bugsnag's [HttpDelivery]. */ +internal object OkHttpSyncHttpDelivery : HttpDelivery { + private val LOGGER = LoggerFactory.getLogger(OkHttpSyncHttpDelivery::class.java) + private const val DEFAULT_TIMEOUT = 5000L + private const val ENDPOINT = "https://notify.bugsnag.com" + + override fun deliver(serializer: Serializer, payload: Any, headers: Map) { + val client = + OkHttpClient.Builder() + .callTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS) + .proxyAuthenticator(Authenticator.JAVA_NET_AUTHENTICATOR) + .build() + val request = + Request.Builder() + .url(ENDPOINT) + .headers(headers.toHeaders()) + .post( + object : RequestBody() { + override fun contentType() = "application/json".toMediaType() + + override fun writeTo(sink: BufferedSink) { + sink.outputStream().use { serializer.writeToStream(it, payload) } + } + } + ) + .build() + + try { + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + LOGGER.warn( + "Error not reported to Bugsnag - got non-200 response code: {}", + response.code + ) + } + } + } catch (ex: IOException) { + LOGGER.warn("Error not reported to Bugsnag - exception when making request", ex) + } + } + + override fun close() { + // Nothing to do here. + } + + override fun setEndpoint(endpoint: String) { + // Unsupported here + } + + override fun setTimeout(timeout: Int) { + // Unsupported here + } + + override fun setProxy(proxy: Proxy) { + // Unsupported here + } +} diff --git a/src/main/kotlin/slack/cli/exec/ProcessedExecCli.kt b/src/main/kotlin/slack/cli/exec/ProcessedExecCli.kt new file mode 100644 index 0000000..c3817d4 --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/ProcessedExecCli.kt @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.multiple +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import com.squareup.moshi.adapter +import eu.jrie.jetbrains.kotlinshell.shell.shell +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.createDirectories +import kotlin.io.path.createTempFile +import kotlin.io.path.deleteRecursively +import kotlin.system.exitProcess +import okio.buffer +import okio.source +import slack.cli.projectDirOption + +/** + * Executes a command with Bugsnag tracing and retries as needed. + * + * Example: + * ``` + * $ ./ --bugsnag-key=1234 --verbose --configurationFile config.json ./gradlew build + * ``` + */ +public class ProcessedExecCli : + CliktCommand("Executes a command with Bugsnag tracing and retries as needed.") { + + private val projectDir by projectDirOption() + private val verbose by option("--verbose", "-v").flag() + private val bugsnagKey by option("--bugsnag-key", envvar = "PE_BUGSNAG_KEY") + private val configurationFile by + option("--config", envvar = "PE_CONFIGURATION_FILE") + .path(mustExist = true, canBeFile = true, canBeDir = false) + + private val args by argument().multiple() + + @OptIn(ExperimentalStdlibApi::class, ExperimentalPathApi::class) + override fun run() { + val moshi = ProcessingUtil.newMoshi() + val config = + configurationFile?.let { + echo("Parsing config file '$it'") + it.source().buffer().use { source -> moshi.adapter().fromJson(source) } + } + ?: ProcessedExecConfig() + // The command to be executed + val cmd = args.joinToString(" ") + + // Temporary file for command output + val tmpDir = projectDir.resolve("tmp/processed_exec") + tmpDir.createDirectories() + + // Initial command execution + val (exitCode, logFile) = executeCommand(cmd, tmpDir) + while (exitCode != 0) { + echo("Command failed with exit code $exitCode. Running processor script...") + + echo("Processing CI failure") + val resultProcessor = ResultProcessor(verbose, bugsnagKey, config, ::echo) + + when (val retrySignal = resultProcessor.process(logFile, false)) { + is RetrySignal.Ack, + RetrySignal.Unknown -> { + echo("Processor exited with 0, exiting with original exit code...") + break + } + is RetrySignal.RetryDelayed -> { + echo( + "Processor script exited with 2, rerunning the command after ${retrySignal.delay}..." + ) + // TODO add option to reclaim memory? + Thread.sleep(retrySignal.delay.inWholeMilliseconds) + val secondResult = executeCommand(cmd, tmpDir) + if (secondResult.exitCode != 0) { + // Process the second failure, then bounce out + resultProcessor.process(secondResult.outputFile, isAfterRetry = true) + } + } + is RetrySignal.RetryImmediately -> { + echo("Processor script exited with 1, rerunning the command immediately...") + // TODO add option to reclaim memory? + val secondResult = executeCommand(cmd, tmpDir) + if (secondResult.exitCode != 0) { + // Process the second failure, then bounce out + resultProcessor.process(secondResult.outputFile, isAfterRetry = true) + } + } + } + } + + // If we got here, all is well + // Delete the tmp files + tmpDir.deleteRecursively() + exitProcess(exitCode) + } + + // Function to execute command and capture output. Shorthand to the testable top-level function. + private fun executeCommand(command: String, tmpDir: Path) = + executeCommand(projectDir, command, tmpDir, ::echo) +} + +internal data class ProcessResult(val exitCode: Int, val outputFile: Path) + +// Function to execute command and capture output +internal fun executeCommand( + workingDir: Path, + command: String, + tmpDir: Path, + echo: (String) -> Unit, +): ProcessResult { + echo("Running command: '$command'") + + val tmpFile = createTempFile(tmpDir, "processed_exec", ".txt").toAbsolutePath() + + var exitCode = 0 + shell { + // Weird but the only way to set the working dir + shell(dir = workingDir.toFile()) { + // Read the output of the process and write to both stdout and file + // This makes it behave a bit like tee. + val echoHandler = stringLambda { line -> + // The line always includes a trailing newline, but we don't need that + echo(line.removeSuffix("\n")) + // Pass the line through unmodified + line to "" + } + val process = command.process() + pipeline { process pipe echoHandler pipe tmpFile.toFile() }.join() + exitCode = process.process.pcb.exitCode + } + } + + return ProcessResult(exitCode, tmpFile) +} diff --git a/src/main/kotlin/slack/cli/exec/ProcessedExecConfig.kt b/src/main/kotlin/slack/cli/exec/ProcessedExecConfig.kt new file mode 100644 index 0000000..c6120dd --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/ProcessedExecConfig.kt @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass +import kotlin.reflect.full.declaredMemberProperties + +private const val CURRENT_VERSION = 1 + +/** Represents a configuration for a processed exec. */ +@JsonClass(generateAdapter = true) +internal data class ProcessedExecConfig( + val version: Int = CURRENT_VERSION, + @Json(name = "gradle_enterprise_server") val gradleEnterpriseServer: String? = null, + @Json(name = "known_issues") + val knownIssues: List = + KnownIssues::class.declaredMemberProperties.map { it.get(KnownIssues) as Issue }, +) { + init { + check(version == CURRENT_VERSION) { + "Incompatible config version. Found $version, expected $CURRENT_VERSION." + } + } +} diff --git a/src/main/kotlin/slack/cli/exec/ProcessingUtil.kt b/src/main/kotlin/slack/cli/exec/ProcessingUtil.kt new file mode 100644 index 0000000..86c12b4 --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/ProcessingUtil.kt @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.squareup.moshi.Moshi +import com.squareup.moshi.addAdapter +import slack.cli.util.DurationJsonAdapter + +internal object ProcessingUtil { + @OptIn(ExperimentalStdlibApi::class) + fun newMoshi(): Moshi { + return Moshi.Builder().addAdapter(DurationJsonAdapter()).build() + } +} diff --git a/src/main/kotlin/slack/cli/exec/ResultProcessor.kt b/src/main/kotlin/slack/cli/exec/ResultProcessor.kt new file mode 100644 index 0000000..25f1434 --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/ResultProcessor.kt @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.bugsnag.Bugsnag +import com.bugsnag.Report +import com.bugsnag.Severity +import java.nio.file.Path +import kotlin.io.path.absolutePathString +import kotlin.io.path.name +import kotlin.io.path.readLines + +/** + * Utility that processes a CI failure log file and optionally signals to retry. + * + * ## Processing + * + * This processes the failure and logs it to Bugsnag for grouping. This is important for us to try + * to track and group failures over time. This is not yet implemented. + * + * ## Signaling + * + * Some CI failures are transient and can be retried. This CLI can signal to retry a CI job by + * exiting with a specific exit code. + * + * Retry signals are + * - exit 0: nothing to do + * - exit 1: retry immediately + * - exit 2: retry in 1 minute + */ +internal class ResultProcessor( + private val verbose: Boolean, + private val bugsnagKey: String?, + private val config: ProcessedExecConfig, + private val echo: (String) -> Unit, +) { + + fun process(logFile: Path, isAfterRetry: Boolean): RetrySignal { + echo("Processing CI log from ${logFile.absolutePathString()}") + + val bugsnag: Bugsnag? by lazy { bugsnagKey?.let { key -> createBugsnag(key) } } + + val logLinesReversed = logFile.readLines().asReversed() + for (issue in config.knownIssues) { + val retrySignal = issue.check(logLinesReversed, echo) + + if (retrySignal != RetrySignal.Unknown) { + // Report to bugsnag. Shared common Throwable but with different messages. + bugsnag?.apply { + verboseEcho("Reporting to bugsnag: $retrySignal") + notify(IssueThrowable(issue), Severity.ERROR) { report -> + // Group by the throwable message + report.setGroupingHash(issue.groupingHash) + report.addToTab("Run Info", "After-Retry", isAfterRetry) + config.gradleEnterpriseServer?.let(logLinesReversed::parseBuildScan)?.let { scanLink -> + report.addToTab("Run Info", "Build-Scan", scanLink) + } + } + } + + if (retrySignal is RetrySignal.Ack) { + echo("Recognized known issue but cannot retry: ${issue.message}") + } else { + echo("Found retry signal: $retrySignal") + } + return retrySignal + } + } + + // TODO some day log these into bugsnag too? + echo("No actionable items found in ${logFile.name}") + return RetrySignal.Unknown + } + + private fun verboseEcho(message: String) { + if (verbose) echo(message) + } + + private fun createBugsnag(key: String): Bugsnag { + return Bugsnag(key).apply { + setAutoCaptureSessions(false) + startSession() + + // Version of this processor for easier tracking of versions. + setAppVersion("1.0.0") + + // Report synchronously. This is a CLI so we don't care about blocking. + // Use our own OkHttp based delivery for better reliability and proxy support. + delivery = OkHttpSyncHttpDelivery + + // Set the app type to the step key. Useful for grouping these by different steps they + // occur in. + envOrNull("BUILDKITE_STEP_KEY")?.let { step -> setAppType(step) } + + // Set the release stage based on the branch name. This lets us slice them by "where" in + // the dev cycle they are occurring. + envOrNull("BUILDKITE_BRANCH")?.let { branch -> + val releaseStage = + when { + branch == "main" -> "main" + // Merge queue branch prefixes in aviator and github + branch.startsWith("mq-") || branch.startsWith("gh-readonly-queue") -> "merge-queue" + else -> "pull-request" + } + setReleaseStage(releaseStage) + } + + // Add metadata to reports + addCallback { report -> + verboseEcho("Adding metadata to report") + + // Tabs with misc build info. + report.populateDeviceTab() + report.populateBuildKiteTab() + } + } + } +} + +private fun Report.populateDeviceTab() { + addToTab("Device", "OS-version", System.getProperty("os.version")) + addToTab("Device", "JRE", System.getProperty("java.version")) + addToTab("Device", "Kotlin", KotlinVersion.CURRENT.toString()) +} + +private fun Report.populateBuildKiteTab() { + envOrNull("BUILDKITE_JOB_ID")?.let { jobId -> addToTab("BuildKite", "Job-ID", jobId) } + envOrNull("BUILDKITE_BUILD_ID")?.let { addToTab("BuildKite", "ID", it) } + envOrNull("BUILDKITE_BUILD_URL")?.let { addToTab("BuildKite", "URL", it) } + envOrNull("BUILDKITE_STEP_KEY")?.let { addToTab("BuildKite", "Step-Key", it) } + envOrNull("BUILDKITE_COMMAND")?.let { addToTab("BuildKite", "CI-Command", it) } +} + +private fun envOrNull(envKey: String) = System.getenv(envKey)?.takeUnless { it.isBlank() } + +internal fun List.parseBuildScan(serverUrl: String): String? { + // Find a build scan URL like so + // Publishing build scan... + // https://some-server.com/s/ueizlbptdqv6q + + // Index of the publish log. Scan link should be above or below this. + val indexOfBuildScan = indexOfFirst { it.contains("Publishing build scan...") } + // Note the lines may be in reverse order here, so try both above and below + return get(indexOfBuildScan - 1).trim().takeIf { serverUrl in it } + ?: get(indexOfBuildScan + 1).trim().takeIf { serverUrl in it } +} diff --git a/src/main/kotlin/slack/cli/exec/RetrySignal.kt b/src/main/kotlin/slack/cli/exec/RetrySignal.kt new file mode 100644 index 0000000..37a16ce --- /dev/null +++ b/src/main/kotlin/slack/cli/exec/RetrySignal.kt @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.squareup.moshi.JsonClass +import dev.zacsweers.moshix.sealed.annotations.TypeLabel +import kotlin.time.Duration + +@JsonClass(generateAdapter = true, generator = "sealed:type") +internal sealed interface RetrySignal { + + /** Unknown issue. */ + @TypeLabel("unknown") object Unknown : RetrySignal + + /** Indicates an issue that is recognized but cannot be retried. */ + @TypeLabel("ack") object Ack : RetrySignal + + /** Indicates this issue should be retried immediately. */ + @TypeLabel("immediate") object RetryImmediately : RetrySignal + + /** Indicates this issue should be retried after a [delay]. */ + @TypeLabel("delayed") + @JsonClass(generateAdapter = true) + data class RetryDelayed( + // Can't default to 1.minutes due to https://github.com/ZacSweers/MoshiX/issues/442 + val delay: Duration + ) : RetrySignal +} diff --git a/src/main/kotlin/slack/cli/util/DurationJsonAdapter.kt b/src/main/kotlin/slack/cli/util/DurationJsonAdapter.kt new file mode 100644 index 0000000..67d8a25 --- /dev/null +++ b/src/main/kotlin/slack/cli/util/DurationJsonAdapter.kt @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.util + +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.JsonReader +import com.squareup.moshi.JsonWriter +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds + +/** A simple [Duration] adapter that converts a [Long] (in millis) to a [Duration]. */ +internal class DurationJsonAdapter : JsonAdapter() { + override fun fromJson(reader: JsonReader): Duration { + return reader.nextLong().milliseconds + } + + override fun toJson(writer: JsonWriter, value: Duration?) { + writer.value(value?.inWholeMilliseconds) + } +} diff --git a/src/test/kotlin/slack/cli/exec/ProcessedExecConfigTest.kt b/src/test/kotlin/slack/cli/exec/ProcessedExecConfigTest.kt new file mode 100644 index 0000000..3f0f51f --- /dev/null +++ b/src/test/kotlin/slack/cli/exec/ProcessedExecConfigTest.kt @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.google.common.truth.Truth.assertThat +import com.squareup.moshi.adapter +import kotlin.time.Duration.Companion.minutes +import org.junit.Test + +class ProcessedExecConfigTest { + @OptIn(ExperimentalStdlibApi::class) + @Test + fun simpleParse() { + val adapter = ProcessingUtil.newMoshi().adapter() + // language=json + val json = + """ + { + "version": 1, + "gradle_enterprise_server": "https://gradle-enterprise.example.com", + "known_issues": [ + { + "message": "${KnownIssues.ftlRateLimit.message}", + "log_message": "${KnownIssues.ftlRateLimit.logMessage}", + "matching_text": "${KnownIssues.ftlRateLimit.matchingText}", + "grouping_hash": "${KnownIssues.ftlRateLimit.groupingHash}", + "retry_signal": { + "type": "delayed", + "delay": ${1.minutes.inWholeMilliseconds} + } + } + ] + } + """ + .trimIndent() + + val issue = adapter.fromJson(json)!! + assertThat(issue) + .isEqualTo( + ProcessedExecConfig( + 1, + "https://gradle-enterprise.example.com", + listOf(KnownIssues.ftlRateLimit) + ) + ) + } + + @Test + fun defaults() { + val defaultConfig = ProcessedExecConfig() + assertThat(defaultConfig.knownIssues).isNotEmpty() + } +} diff --git a/src/test/kotlin/slack/cli/exec/ResultProcessorTest.kt b/src/test/kotlin/slack/cli/exec/ResultProcessorTest.kt new file mode 100644 index 0000000..e109b18 --- /dev/null +++ b/src/test/kotlin/slack/cli/exec/ResultProcessorTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2023 Slack Technologies, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package slack.cli.exec + +import com.google.common.truth.Truth.assertThat +import kotlin.io.path.readText +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class ResultProcessorTest { + + @JvmField @Rule val tmpFolder = TemporaryFolder() + + private val logs = ArrayDeque() + + @Test + fun testExecuteCommand() { + tmpFolder.newFile("test.txt") + val tmpDir = tmpFolder.newFolder("tmp/processed_exec") + val (exitCode, outputFile) = + executeCommand(tmpFolder.root.toPath(), "ls -1", tmpDir.toPath(), logs::add) + assertThat(exitCode).isEqualTo(0) + + val expectedOutput = + """ + test.txt + tmp + """ + .trimIndent() + + assertThat(outputFile.readText().trim()).isEqualTo(expectedOutput) + + // Note we use "contains" here because our script may output additional logs + assertThat(logs.joinToString("\n").trim()).contains(expectedOutput) + } + + @Test + fun unknownIssue() { + val outputFile = tmpFolder.newFile("logs.txt") + outputFile.writeText( + """ + [1/2] FAILURE: Build failed with an exception. + """ + .trimIndent() + .padWithTestLogs() + ) + val signal = newProcessor().process(outputFile.toPath(), isAfterRetry = false) + check(signal is RetrySignal.Unknown) + } + + @Test + fun retryDelayed() { + val outputFile = tmpFolder.newFile("logs.txt") + outputFile.writeText( + """ + ${KnownIssues.ftlRateLimit.matchingText} + """.trimIndent().padWithTestLogs() + ) + val signal = newProcessor().process(outputFile.toPath(), isAfterRetry = false) + check(signal is RetrySignal.RetryDelayed) + } + + @Test + fun retryImmediately() { + val outputFile = tmpFolder.newFile("logs.txt") + outputFile.writeText( + """ + ${KnownIssues.oom.matchingText} + """.trimIndent().padWithTestLogs() + ) + val signal = newProcessor().process(outputFile.toPath(), isAfterRetry = false) + check(signal is RetrySignal.RetryImmediately) + } + + @Test + fun ack() { + val outputFile = tmpFolder.newFile("logs.txt") + outputFile.writeText( + """ + ${KnownIssues.fakeFailure.matchingText} + """.trimIndent().padWithTestLogs() + ) + val signal = newProcessor().process(outputFile.toPath(), isAfterRetry = false) + check(signal is RetrySignal.Ack) + } + + @Test + fun parseBuildScan() { + val url = "https://gradle-enterprise.example.com" + val scanUrl = "$url/s/ueizlbptdqv6q" + val log = + """ + Publishing build scan... + $scanUrl + + """.trimIndent().padWithTestLogs() + + // Assert in both directions they match + assertThat(log.lines().parseBuildScan(url)).isEqualTo(scanUrl) + assertThat(log.lines().reversed().parseBuildScan(url)).isEqualTo(scanUrl) + } + + private fun newProcessor(): ResultProcessor { + return ResultProcessor( + verbose = true, + bugsnagKey = null, + config = ProcessedExecConfig(), + echo = logs::add + ) + } + + // Helper to ensure we're parsing logs from within the test output + private fun String.padWithTestLogs(): String { + val prefix = (1..10).joinToString("\n") { randomString() } + val suffix = (1..10).joinToString("\n") { randomString() } + return "$prefix\n${randomString()}$this${randomString()}\n$suffix" + } + + private fun randomString(): String { + return (0..10).map { ('a'..'z').random() }.joinToString("") + } +}