From fb49c72585ee1b780081de01c3dbeb4091207ea3 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 14:20:21 -0400 Subject: [PATCH 1/9] fix gradle cache issue --- .../smithy-kotlin-codegen/build.gradle.kts | 20 ++--- .../test-suite/build.gradle.kts | 73 ++++++++++--------- .../serde-benchmarks/build.gradle.kts | 44 ++++++----- tests/codegen/serde-tests/build.gradle.kts | 37 +++++++--- 4 files changed, 101 insertions(+), 73 deletions(-) diff --git a/codegen/smithy-kotlin-codegen/build.gradle.kts b/codegen/smithy-kotlin-codegen/build.gradle.kts index 0ab1c9881b..377fe86049 100644 --- a/codegen/smithy-kotlin-codegen/build.gradle.kts +++ b/codegen/smithy-kotlin-codegen/build.gradle.kts @@ -20,8 +20,7 @@ val codegenVersion: String by project group = "software.amazon.smithy.kotlin" version = codegenVersion -val sdkVersion: String by project -val runtimeVersion = sdkVersion +val runtimeVersion: Provider = providers.gradleProperty("sdkVersion") dependencies { api(libs.smithy.codegen.core) @@ -42,16 +41,17 @@ dependencies { } val generateSdkRuntimeVersion by tasks.registering { - // generate the version of the runtime to use as a resource. - // this keeps us from having to manually change version numbers in multiple places - val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core").get() - val versionFile = file("$resourcesDir/sdk-version.txt") - val gradlePropertiesFile = rootProject.file("gradle.properties") - inputs.file(gradlePropertiesFile) + val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core") + val versionFile = resourcesDir.map { it.file("sdk-version.txt") } + + // declare inputs/outputs via providers + inputs.property("runtimeVersion", runtimeVersion) outputs.file(versionFile) - sourceSets.main.get().output.dir(resourcesDir) + doLast { - versionFile.writeText(runtimeVersion) + val version = inputs.properties["runtimeVersion"] as String + val file = versionFile.get().asFile + file.writeText(version) } } diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index d2487c2722..b63637fb4c 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -58,30 +58,30 @@ kotlin { } } -open class LocalTestServers : DefaultTask() { - @Internal - var server: Closeable? = null - private set +val osName = System.getProperty("os.name") - @Internal - lateinit var main: String +abstract class TestServerService : + BuildService, + AutoCloseable { + interface Params : BuildServiceParameters { + val sslConfigPath: Property + val mainClass: Property + val classpath: ConfigurableFileCollection + } - @Internal - lateinit var classpath: FileCollection + private var server: Closeable? = null - @Input - lateinit var sslConfigPath: String + fun startServers() { + if (server != null) return - @TaskAction - fun exec() { try { println("[TestServers] start") - val urlClassLoaderSource = classpath.map { file -> file.toURI().toURL() }.toTypedArray() + val urlClassLoaderSource = parameters.classpath.map { it.toURI().toURL() }.toTypedArray() val loader = URLClassLoader(urlClassLoaderSource, ClassLoader.getSystemClassLoader()) - val mainClass = loader.loadClass(main) - val main = mainClass.getMethod("startServers", String::class.java) - server = main.invoke(null, sslConfigPath) as Closeable + val mainClass = loader.loadClass(parameters.mainClass.get()) + val method = mainClass.getMethod("startServers", String::class.java) + server = method.invoke(null, parameters.sslConfigPath.get()) as Closeable println("[TestServers] started") } catch (cause: Throwable) { println("[TestServers] failed: ${cause.message}") @@ -89,36 +89,47 @@ open class LocalTestServers : DefaultTask() { } } - fun stop() { - if (server != null) { - server?.close() - println("[TestServers] stop") - } + override fun close() { + server?.close() + println("[TestServers] stopped") } } -val osName = System.getProperty("os.name") +val testServerService = gradle.sharedServices.registerIfAbsent("testServers", TestServerService::class) { + parameters.sslConfigPath.set(File.createTempFile("ssl-", ".cfg").absolutePath) + parameters.mainClass.set("aws.smithy.kotlin.runtime.http.test.util.TestServersKt") + val kotlinCompilation = kotlin.targets.getByName("jvm").compilations["test"] + parameters.classpath.from(kotlinCompilation.runtimeDependencyFiles!!) +} -val startTestServers = task("startTestServers") { - dependsOn(tasks["jvmJar"]) +abstract class StartTestServersTask : DefaultTask() { + @get:Internal + abstract val serverService: Property - main = "aws.smithy.kotlin.runtime.http.test.util.TestServersKt" - val kotlinCompilation = kotlin.targets.getByName("jvm").compilations["test"] - classpath = kotlinCompilation.runtimeDependencyFiles!! - sslConfigPath = File.createTempFile("ssl-", ".cfg").absolutePath + @TaskAction + fun start() { + serverService.get().startServers() + } +} + +val startTestServers = tasks.register("startTestServers") { + dependsOn(tasks["jvmJar"]) + usesService(testServerService) + serverService.set(testServerService) } val testTasks = listOf("allTests", "jvmTest") .forEach { tasks.named(it) { dependsOn(startTestServers) + usesService(testServerService) } } tasks.jvmTest { // set test environment for proxy tests systemProperty("MITM_PROXY_SCRIPTS_ROOT", projectDir.resolve("proxy-scripts").absolutePath) - systemProperty("SSL_CONFIG_PATH", startTestServers.sslConfigPath) + systemProperty("SSL_CONFIG_PATH", testServerService.get().parameters.sslConfigPath) val enableProxyTestsProp = "aws.test.http.enableProxyTests" val runningInCodeBuild = System.getenv().containsKey("CODEBUILD_BUILD_ID") @@ -127,7 +138,3 @@ tasks.jvmTest { systemProperty(enableProxyTestsProp, System.getProperties().getOrDefault(enableProxyTestsProp, shouldRunProxyTests)) } - -gradle.buildFinished { - startTestServers.stop() -} diff --git a/tests/benchmarks/serde-benchmarks/build.gradle.kts b/tests/benchmarks/serde-benchmarks/build.gradle.kts index a604a67ecf..977a4e4ca3 100644 --- a/tests/benchmarks/serde-benchmarks/build.gradle.kts +++ b/tests/benchmarks/serde-benchmarks/build.gradle.kts @@ -85,27 +85,27 @@ tasks.generateSmithyProjections { buildClasspath.set(codegen) } -data class BenchmarkModel(val name: String) { - val projectionRootDir: File - get() = layout.buildDirectory.dir("smithyprojections/${project.name}/$name/kotlin-codegen/src/main/kotlin").get().asFile.absoluteFile +abstract class StageGeneratedSourcesTask : DefaultTask() { + @get:Input + abstract val projectName: Property - val sourceSetRootDir: File - get() = layout.buildDirectory.dir("generated-src/src").get().asFile.absoluteFile -} + @get:InputDirectory + abstract val smithyProjectionsDir: DirectoryProperty -val benchmarkModels = listOf( - "twitter", - "countries-states", -).map { BenchmarkModel(it) } + @get:OutputDirectory + abstract val generatedSourcesDir: DirectoryProperty -val stageGeneratedSources = tasks.register("stageGeneratedSources") { - group = "codegen" - dependsOn(tasks.generateSmithyProjections) - doLast { - benchmarkModels.forEach { - copy { - from("${it.projectionRootDir}") - into("${it.sourceSetRootDir}") + @get:Inject + abstract val fs: FileSystemOperations + + @TaskAction + fun stage() { + val models = listOf("twitter", "countries-states") + + models.forEach { modelName -> + fs.copy { + from("$smithyProjectionsDir/$projectName/$modelName/kotlin-codegen/src/main/kotlin") + into(generatedSourcesDir.get().asFile) include("**/model/*.kt") include("**/serde/*.kt") exclude("**/serde/*OperationSerializer.kt") @@ -115,6 +115,14 @@ val stageGeneratedSources = tasks.register("stageGeneratedSources") { } } +val stageGeneratedSources = tasks.register("stageGeneratedSources") { + group = "codegen" + dependsOn(tasks.generateSmithyProjections) + projectName.set(project.name) + smithyProjectionsDir.set(layout.buildDirectory.dir("smithyprojections")) + generatedSourcesDir.set(layout.buildDirectory.dir("generated-src/src")) +} + tasks.withType { dependsOn(stageGeneratedSources) } diff --git a/tests/codegen/serde-tests/build.gradle.kts b/tests/codegen/serde-tests/build.gradle.kts index fac6eb51ca..3b8d3f3eba 100644 --- a/tests/codegen/serde-tests/build.gradle.kts +++ b/tests/codegen/serde-tests/build.gradle.kts @@ -58,18 +58,24 @@ dependencies { val generatedSrcDir = project.layout.projectDirectory.dir("generated-src/main/kotlin") -val stageGeneratedSources = tasks.register("stageGeneratedSources") { - group = "codegen" - dependsOn(tasks.generateSmithyProjections) - outputs.dir(generatedSrcDir) - // FIXME - this task up-to-date checks are wrong, likely something is not setup right with inputs/outputs somewhere - // for now just always run it - outputs.upToDateWhen { false } - doLast { - listOf("xml", "json").forEach { projectionName -> - val fromDir = smithyBuild.smithyKotlinProjectionSrcDir(projectionName) - logger.info("copying from ${fromDir.get()} to $generatedSrcDir") - copy { +abstract class StageGeneratedSourcesTask : DefaultTask() { + @get:InputDirectory + abstract val projectionsDir: DirectoryProperty + + @get:OutputDirectory + abstract val generatedSrcDir: DirectoryProperty + + @get:Inject + abstract val fs: FileSystemOperations + + @TaskAction + fun stageGeneratedSources() { + val projections = listOf("xml", "json") + + projections.forEach { projectionName -> + val fromDir = projectionsDir.dir("$projectionName/kotlin-codegen/src/main/kotlin") + logger.info("copying from ${fromDir.get()} to ${generatedSrcDir.get()}") + fs.copy { from(fromDir) into(generatedSrcDir) include("**/model/*.kt") @@ -83,6 +89,13 @@ val stageGeneratedSources = tasks.register("stageGeneratedSources") { } } +val stageGeneratedSources = tasks.register("stageGeneratedSources") { + group = "codegen" + dependsOn(tasks.generateSmithyProjections) + generatedSrcDir.set(layout.projectDirectory.dir("generated-src/main/kotlin")) + projectionsDir.set(layout.buildDirectory.dir("smithyprojections/serde-tests")) +} + tasks.kotlinSourcesJar { dependsOn( tasks.generateSmithyProjections, From b308e9d59c0a29ef48ba8d6ddc4effe5fd59efa6 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 14:24:58 -0400 Subject: [PATCH 2/9] cleanup --- codegen/smithy-kotlin-codegen/build.gradle.kts | 1 - .../protocol/http-client-engines/test-suite/build.gradle.kts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/codegen/smithy-kotlin-codegen/build.gradle.kts b/codegen/smithy-kotlin-codegen/build.gradle.kts index 377fe86049..6362f70f15 100644 --- a/codegen/smithy-kotlin-codegen/build.gradle.kts +++ b/codegen/smithy-kotlin-codegen/build.gradle.kts @@ -44,7 +44,6 @@ val generateSdkRuntimeVersion by tasks.registering { val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core") val versionFile = resourcesDir.map { it.file("sdk-version.txt") } - // declare inputs/outputs via providers inputs.property("runtimeVersion", runtimeVersion) outputs.file(versionFile) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index b63637fb4c..feb4171cba 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -58,8 +58,6 @@ kotlin { } } -val osName = System.getProperty("os.name") - abstract class TestServerService : BuildService, AutoCloseable { @@ -112,6 +110,8 @@ abstract class StartTestServersTask : DefaultTask() { } } +val osName = System.getProperty("os.name") + val startTestServers = tasks.register("startTestServers") { dependsOn(tasks["jvmJar"]) usesService(testServerService) From 9816fb264ee8b2bb23692402fe9081a0811b06ff Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 14:30:43 -0400 Subject: [PATCH 3/9] fix --- .../protocol/http-client-engines/test-suite/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index feb4171cba..4a31670d5a 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -129,7 +129,7 @@ val testTasks = listOf("allTests", "jvmTest") tasks.jvmTest { // set test environment for proxy tests systemProperty("MITM_PROXY_SCRIPTS_ROOT", projectDir.resolve("proxy-scripts").absolutePath) - systemProperty("SSL_CONFIG_PATH", testServerService.get().parameters.sslConfigPath) + systemProperty("SSL_CONFIG_PATH", testServerService.get().parameters.sslConfigPath.get()) val enableProxyTestsProp = "aws.test.http.enableProxyTests" val runningInCodeBuild = System.getenv().containsKey("CODEBUILD_BUILD_ID") From 7da0bedbab871457a03d52a07d8c5f092f5947c2 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 14:32:13 -0400 Subject: [PATCH 4/9] naming --- .../protocol/http-client-engines/test-suite/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index 4a31670d5a..e036e01093 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -78,8 +78,8 @@ abstract class TestServerService : val loader = URLClassLoader(urlClassLoaderSource, ClassLoader.getSystemClassLoader()) val mainClass = loader.loadClass(parameters.mainClass.get()) - val method = mainClass.getMethod("startServers", String::class.java) - server = method.invoke(null, parameters.sslConfigPath.get()) as Closeable + val main = mainClass.getMethod("startServers", String::class.java) + server = main.invoke(null, parameters.sslConfigPath.get()) as Closeable println("[TestServers] started") } catch (cause: Throwable) { println("[TestServers] failed: ${cause.message}") From ddca24a708df8afa071b369c989d2b9db33beade Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 14:52:12 -0400 Subject: [PATCH 5/9] fix --- tests/benchmarks/serde-benchmarks/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/serde-benchmarks/build.gradle.kts b/tests/benchmarks/serde-benchmarks/build.gradle.kts index 977a4e4ca3..9bad81869e 100644 --- a/tests/benchmarks/serde-benchmarks/build.gradle.kts +++ b/tests/benchmarks/serde-benchmarks/build.gradle.kts @@ -104,7 +104,7 @@ abstract class StageGeneratedSourcesTask : DefaultTask() { models.forEach { modelName -> fs.copy { - from("$smithyProjectionsDir/$projectName/$modelName/kotlin-codegen/src/main/kotlin") + from(smithyProjectionsDir.dir("${projectName.get()}/$modelName/kotlin-codegen/src/main/kotlin")) into(generatedSourcesDir.get().asFile) include("**/model/*.kt") include("**/serde/*.kt") From 49921898b0b138d2cf7d351b7d9339f031cf01c7 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 15:25:10 -0400 Subject: [PATCH 6/9] fix --- .../http-client-engines/test-suite/build.gradle.kts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index e036e01093..2a11d263c0 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -96,8 +96,10 @@ abstract class TestServerService : val testServerService = gradle.sharedServices.registerIfAbsent("testServers", TestServerService::class) { parameters.sslConfigPath.set(File.createTempFile("ssl-", ".cfg").absolutePath) parameters.mainClass.set("aws.smithy.kotlin.runtime.http.test.util.TestServersKt") - val kotlinCompilation = kotlin.targets.getByName("jvm").compilations["test"] - parameters.classpath.from(kotlinCompilation.runtimeDependencyFiles!!) +} + +afterEvaluate { + testServerService.get().parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!) } abstract class StartTestServersTask : DefaultTask() { From fb1f2e19043e7ce6ced9216410064ab4dc2f523b Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Fri, 26 Sep 2025 16:18:20 -0400 Subject: [PATCH 7/9] fix --- tests/benchmarks/serde-benchmarks/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/benchmarks/serde-benchmarks/build.gradle.kts b/tests/benchmarks/serde-benchmarks/build.gradle.kts index 9bad81869e..8e237a256d 100644 --- a/tests/benchmarks/serde-benchmarks/build.gradle.kts +++ b/tests/benchmarks/serde-benchmarks/build.gradle.kts @@ -123,6 +123,12 @@ val stageGeneratedSources = tasks.register("stageGene generatedSourcesDir.set(layout.buildDirectory.dir("generated-src/src")) } +afterEvaluate { + tasks.named("jvmSourcesJar") { + dependsOn(stageGeneratedSources) + } +} + tasks.withType { dependsOn(stageGeneratedSources) } From 72cdffc2ee1452f1b0c2501e53a1900f92442f1e Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 29 Sep 2025 12:23:46 -0400 Subject: [PATCH 8/9] pr feedback --- .../test-suite/build.gradle.kts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index 2a11d263c0..e904fc66fb 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -58,12 +58,11 @@ kotlin { } } -abstract class TestServerService : - BuildService, +abstract class TestServerProvider : + BuildService, AutoCloseable { interface Params : BuildServiceParameters { val sslConfigPath: Property - val mainClass: Property val classpath: ConfigurableFileCollection } @@ -77,7 +76,7 @@ abstract class TestServerService : val urlClassLoaderSource = parameters.classpath.map { it.toURI().toURL() }.toTypedArray() val loader = URLClassLoader(urlClassLoaderSource, ClassLoader.getSystemClassLoader()) - val mainClass = loader.loadClass(parameters.mainClass.get()) + val mainClass = loader.loadClass("aws.smithy.kotlin.runtime.http.test.util.TestServersKt") val main = mainClass.getMethod("startServers", String::class.java) server = main.invoke(null, parameters.sslConfigPath.get()) as Closeable println("[TestServers] started") @@ -89,26 +88,23 @@ abstract class TestServerService : override fun close() { server?.close() + server = null println("[TestServers] stopped") } } -val testServerService = gradle.sharedServices.registerIfAbsent("testServers", TestServerService::class) { +val testServerProvider = gradle.sharedServices.registerIfAbsent("testServers", TestServerProvider::class) { parameters.sslConfigPath.set(File.createTempFile("ssl-", ".cfg").absolutePath) - parameters.mainClass.set("aws.smithy.kotlin.runtime.http.test.util.TestServersKt") -} - -afterEvaluate { - testServerService.get().parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!) + parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!) } abstract class StartTestServersTask : DefaultTask() { @get:Internal - abstract val serverService: Property + abstract val serverProvider: Property @TaskAction fun start() { - serverService.get().startServers() + serverProvider.get().startServers() } } @@ -116,22 +112,22 @@ val osName = System.getProperty("os.name") val startTestServers = tasks.register("startTestServers") { dependsOn(tasks["jvmJar"]) - usesService(testServerService) - serverService.set(testServerService) + usesService(testServerProvider) + serverProvider.set(testServerProvider) } val testTasks = listOf("allTests", "jvmTest") .forEach { tasks.named(it) { dependsOn(startTestServers) - usesService(testServerService) + usesService(testServerProvider) } } tasks.jvmTest { // set test environment for proxy tests systemProperty("MITM_PROXY_SCRIPTS_ROOT", projectDir.resolve("proxy-scripts").absolutePath) - systemProperty("SSL_CONFIG_PATH", testServerService.get().parameters.sslConfigPath.get()) + systemProperty("SSL_CONFIG_PATH", testServerProvider.get().parameters.sslConfigPath.get()) val enableProxyTestsProp = "aws.test.http.enableProxyTests" val runningInCodeBuild = System.getenv().containsKey("CODEBUILD_BUILD_ID") From 8ec9e91998d6d94e4d00fc6a50f02fc147be56bd Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 29 Sep 2025 12:40:19 -0400 Subject: [PATCH 9/9] fix --- .../protocol/http-client-engines/test-suite/build.gradle.kts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts index e904fc66fb..833f7a2c8f 100644 --- a/runtime/protocol/http-client-engines/test-suite/build.gradle.kts +++ b/runtime/protocol/http-client-engines/test-suite/build.gradle.kts @@ -95,7 +95,10 @@ abstract class TestServerProvider : val testServerProvider = gradle.sharedServices.registerIfAbsent("testServers", TestServerProvider::class) { parameters.sslConfigPath.set(File.createTempFile("ssl-", ".cfg").absolutePath) - parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!) +} + +afterEvaluate { + testServerProvider.get().parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!) } abstract class StartTestServersTask : DefaultTask() {