Skip to content

misc: enable gradle cache #1427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
Open

misc: enable gradle cache #1427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
fb49c72
fix gradle cache issue
xinsong-cui Sep 26, 2025
b308e9d
cleanup
xinsong-cui Sep 26, 2025
9816fb2
fix
xinsong-cui Sep 26, 2025
7da0bed
naming
xinsong-cui Sep 26, 2025
ddca24a
fix
xinsong-cui Sep 26, 2025
4992189
fix
xinsong-cui Sep 26, 2025
fb1f2e1
fix
xinsong-cui Sep 26, 2025
72cdffc
pr feedback
xinsong-cui Sep 29, 2025
8ec9e91
fix
xinsong-cui Sep 29, 2025
06457f9
merge main
xinsong-cui Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions codegen/smithy-kotlin-codegen/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ val codegenVersion: String by project
group = "software.amazon.smithy.kotlin"
version = codegenVersion

val sdkVersion: String by project
val runtimeVersion = sdkVersion
val runtimeVersion: Provider<String> = providers.gradleProperty("sdkVersion")

dependencies {
api(libs.smithy.codegen.core)
Expand All @@ -42,16 +41,16 @@ dependencies {
}

val generateSdkRuntimeVersion by tasks.registering {
// generate the version of the runtime to use as a resource.
// this keeps us from having to manually change version numbers in multiple places
val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core").get()
val versionFile = file("$resourcesDir/sdk-version.txt")
val gradlePropertiesFile = rootProject.file("gradle.properties")
inputs.file(gradlePropertiesFile)
val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core")
val versionFile = resourcesDir.map { it.file("sdk-version.txt") }

inputs.property("runtimeVersion", runtimeVersion)
outputs.file(versionFile)
sourceSets.main.get().output.dir(resourcesDir)

doLast {
versionFile.writeText(runtimeVersion)
val version = inputs.properties["runtimeVersion"] as String
val file = versionFile.get().asFile
file.writeText(version)
}
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactor to get rid of Gradle script object references

Expand Down
72 changes: 40 additions & 32 deletions runtime/protocol/http-client-engines/test-suite/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,67 +59,79 @@ kotlin {
}
}

open class LocalTestServers : DefaultTask() {
@Internal
var server: Closeable? = null
private set

@Internal
lateinit var main: String
abstract class TestServerProvider :
BuildService<TestServerProvider.Params>,
AutoCloseable {
interface Params : BuildServiceParameters {
val sslConfigPath: Property<String>
val classpath: ConfigurableFileCollection
}

@Internal
lateinit var classpath: FileCollection
private var server: Closeable? = null

@Input
lateinit var sslConfigPath: String
fun startServers() {
if (server != null) return

@TaskAction
fun exec() {
try {
println("[TestServers] start")
val urlClassLoaderSource = classpath.map { file -> file.toURI().toURL() }.toTypedArray()
val urlClassLoaderSource = parameters.classpath.map { it.toURI().toURL() }.toTypedArray()
val loader = URLClassLoader(urlClassLoaderSource, ClassLoader.getSystemClassLoader())

val mainClass = loader.loadClass(main)
val mainClass = loader.loadClass("aws.smithy.kotlin.runtime.http.test.util.TestServersKt")
val main = mainClass.getMethod("startServers", String::class.java)
server = main.invoke(null, sslConfigPath) as Closeable
server = main.invoke(null, parameters.sslConfigPath.get()) as Closeable
println("[TestServers] started")
} catch (cause: Throwable) {
println("[TestServers] failed: ${cause.message}")
throw cause
}
}

fun stop() {
if (server != null) {
server?.close()
println("[TestServers] stop")
}
override fun close() {
server?.close()
server = null
println("[TestServers] stopped")
}
}

val testServerProvider = gradle.sharedServices.registerIfAbsent("testServers", TestServerProvider::class) {
parameters.sslConfigPath.set(File.createTempFile("ssl-", ".cfg").absolutePath)
}

afterEvaluate {
testServerProvider.get().parameters.classpath.from(kotlin.targets.getByName("jvm").compilations["test"].runtimeDependencyFiles!!)
}

abstract class StartTestServersTask : DefaultTask() {
@get:Internal
abstract val serverProvider: Property<TestServerProvider>

@TaskAction
fun start() {
serverProvider.get().startServers()
}
}

val osName = System.getProperty("os.name")

val startTestServers = task<LocalTestServers>("startTestServers") {
val startTestServers = tasks.register<StartTestServersTask>("startTestServers") {
dependsOn(tasks["jvmJar"])

main = "aws.smithy.kotlin.runtime.http.test.util.TestServersKt"
val kotlinCompilation = kotlin.targets.getByName("jvm").compilations["test"]
classpath = kotlinCompilation.runtimeDependencyFiles!!
sslConfigPath = File.createTempFile("ssl-", ".cfg").absolutePath
usesService(testServerProvider)
serverProvider.set(testServerProvider)
}

val testTasks = listOf("allTests", "jvmTest")
.forEach {
tasks.named(it) {
dependsOn(startTestServers)
usesService(testServerProvider)
}
}

tasks.jvmTest {
// set test environment for proxy tests
systemProperty("MITM_PROXY_SCRIPTS_ROOT", projectDir.resolve("proxy-scripts").absolutePath)
systemProperty("SSL_CONFIG_PATH", startTestServers.sslConfigPath)
systemProperty("SSL_CONFIG_PATH", testServerProvider.get().parameters.sslConfigPath.get())

val enableProxyTestsProp = "aws.test.http.enableProxyTests"
val runningInCodeBuild = System.getenv().containsKey("CODEBUILD_BUILD_ID")
Expand All @@ -128,7 +140,3 @@ tasks.jvmTest {

systemProperty(enableProxyTestsProp, System.getProperties().getOrDefault(enableProxyTestsProp, shouldRunProxyTests))
}

gradle.buildFinished {
startTestServers.stop()
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactor to address 'Gradle.buildFinished' is unsupported

Comment on lines -132 to -134
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do the test servers get shut down now, when does TestServerService::close get invoked?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TestServerService is AutoCloseable now, TestServerService::close() will get invoke when build finished. (Same as original version)

50 changes: 32 additions & 18 deletions tests/benchmarks/serde-benchmarks/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,27 @@ tasks.generateSmithyProjections {
buildClasspath.set(codegen)
}

data class BenchmarkModel(val name: String) {
val projectionRootDir: File
get() = layout.buildDirectory.dir("smithyprojections/${project.name}/$name/kotlin-codegen/src/main/kotlin").get().asFile.absoluteFile
abstract class StageGeneratedSourcesTask : DefaultTask() {
@get:Input
abstract val projectName: Property<String>

val sourceSetRootDir: File
get() = layout.buildDirectory.dir("generated-src/src").get().asFile.absoluteFile
}
@get:InputDirectory
abstract val smithyProjectionsDir: DirectoryProperty

val benchmarkModels = listOf(
"twitter",
"countries-states",
).map { BenchmarkModel(it) }
@get:OutputDirectory
abstract val generatedSourcesDir: DirectoryProperty

val stageGeneratedSources = tasks.register("stageGeneratedSources") {
group = "codegen"
dependsOn(tasks.generateSmithyProjections)
doLast {
benchmarkModels.forEach {
copy {
from("${it.projectionRootDir}")
into("${it.sourceSetRootDir}")
@get:Inject
abstract val fs: FileSystemOperations
Comment on lines +98 to +99
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why can't we use the built-in file operations like we do in other Gradle tasks?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

using project.copy accesses the Project instance at task execution time which incompatible with gradle cache


@TaskAction
fun stage() {
val models = listOf("twitter", "countries-states")

models.forEach { modelName ->
fs.copy {
from(smithyProjectionsDir.dir("${projectName.get()}/$modelName/kotlin-codegen/src/main/kotlin"))
into(generatedSourcesDir.get().asFile)
include("**/model/*.kt")
include("**/serde/*.kt")
exclude("**/serde/*OperationSerializer.kt")
Expand All @@ -115,6 +115,20 @@ val stageGeneratedSources = tasks.register("stageGeneratedSources") {
}
}

val stageGeneratedSources = tasks.register<StageGeneratedSourcesTask>("stageGeneratedSources") {
group = "codegen"
dependsOn(tasks.generateSmithyProjections)
projectName.set(project.name)
smithyProjectionsDir.set(layout.buildDirectory.dir("smithyprojections"))
generatedSourcesDir.set(layout.buildDirectory.dir("generated-src/src"))
}

afterEvaluate {
tasks.named("jvmSourcesJar") {
dependsOn(stageGeneratedSources)
}
}
Comment on lines +126 to +130
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need to configure this for JVM tasks? Does this work for Native builds too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was gettting Reason: Task ':tests:benchmarks:serde-benchmarks:jvmSourcesJar' uses this output of task ':tests:benchmarks:serde-benchmarks:stageGeneratedSources' without declaring an explicit or implicit dependency. This can lead to incorrect results being produced, depending on what order the tasks are executed. without it, I am guessing also need this for Native? but not sure

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well I am a couple CI checks away from merging kn-main into main, you should be able to test it out when you rebase

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just merge it, seems like work fines for Native


tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactor to get rid of Gradle script object references

dependsOn(stageGeneratedSources)
}
37 changes: 25 additions & 12 deletions tests/codegen/serde-tests/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,24 @@ dependencies {

val generatedSrcDir = project.layout.projectDirectory.dir("generated-src/main/kotlin")

val stageGeneratedSources = tasks.register("stageGeneratedSources") {
group = "codegen"
dependsOn(tasks.generateSmithyProjections)
outputs.dir(generatedSrcDir)
// FIXME - this task up-to-date checks are wrong, likely something is not setup right with inputs/outputs somewhere
// for now just always run it
outputs.upToDateWhen { false }
doLast {
listOf("xml", "json").forEach { projectionName ->
val fromDir = smithyBuild.smithyKotlinProjectionSrcDir(projectionName)
logger.info("copying from ${fromDir.get()} to $generatedSrcDir")
copy {
abstract class StageGeneratedSourcesTask : DefaultTask() {
@get:InputDirectory
abstract val projectionsDir: DirectoryProperty

@get:OutputDirectory
abstract val generatedSrcDir: DirectoryProperty

@get:Inject
abstract val fs: FileSystemOperations

@TaskAction
fun stageGeneratedSources() {
val projections = listOf("xml", "json")

projections.forEach { projectionName ->
val fromDir = projectionsDir.dir("$projectionName/kotlin-codegen/src/main/kotlin")
logger.info("copying from ${fromDir.get()} to ${generatedSrcDir.get()}")
fs.copy {
from(fromDir)
into(generatedSrcDir)
include("**/model/*.kt")
Expand All @@ -83,6 +89,13 @@ val stageGeneratedSources = tasks.register("stageGeneratedSources") {
}
}

val stageGeneratedSources = tasks.register<StageGeneratedSourcesTask>("stageGeneratedSources") {
group = "codegen"
dependsOn(tasks.generateSmithyProjections)
generatedSrcDir.set(layout.projectDirectory.dir("generated-src/main/kotlin"))
projectionsDir.set(layout.buildDirectory.dir("smithyprojections/serde-tests"))
}

tasks.kotlinSourcesJar {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactor to get rid of Gradle script object references

dependsOn(
tasks.generateSmithyProjections,
Expand Down
Loading