build.gradle

import com.github.spotbugs.snom.SpotBugsTask
import ru.vyarus.gradle.plugin.python.task.PythonTask
import com.hierynomus.gradle.license.tasks.LicenseCheck
import com.hierynomus.gradle.license.tasks.LicenseFormat

// The buildscript block defines dependencies in order for .gradle file evaluation.
// This is separate from application dependencies.
// See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle.
buildscript {
    repositories {
        maven {
            url "https://plugins.gradle.org/m2/"
        }
    }
    dependencies {
        classpath 'com.bmuschko:gradle-docker-plugin:8.0.0'
        // 6.x version of OpenApi generator is only compatible with jackson-core 2.13.x onwards.
        // This conflicts with the jackson depencneis the bmuschko plugin is pulling in.
        // Since api generation is only used in the airbyte-api module and the base gradle files
        // are loaded in first, Gradle is not able to intelligently resolve this before loading in
        // the bmuschko plugin and thus placing an older jackson version on the class path.
        // The alternative is to import the openapi plugin for all modules.
        // This might need to be updated when we change openapi plugin versions.
        classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0'

        classpath 'org.codehaus.groovy:groovy-yaml:3.0.3'

        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.20.0'
        classpath 'gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.16.1'
    }
}

plugins {
    id 'base'
    id 'com.github.node-gradle.node' version '3.5.1'
    id 'com.github.spotbugs' version '5.0.13'
    id 'version-catalog'
    id 'ru.vyarus.use-python'
}


Properties env = new Properties()
rootProject.file('gradle.properties').withInputStream { env.load(it) }

if (!env.containsKey('VERSION')) {
    throw new Exception('Version not specified in .env file...')
}

// `version` is used as the application build version for artifacts like jars
// `image_tag` is used as the docker tag applied to built images.
// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
ext {
    version = System.getenv("VERSION") ?: env.VERSION
    image_tag = System.getenv("VERSION") ?: 'dev'
    skipSlowTests = (System.getProperty('skipSlowTests', 'false') != 'false')
}

def isConnectorProject = { Project project ->
    if (project.parent == null || project.parent.name != 'connectors') {
        return false
    }
    return project.name.startsWith("source-") || project.name.startsWith("destination-")
}

allprojects {
    apply plugin: 'base'

    // by default gradle uses directory as the project name. That works very well in a single project environment but
    // projects clobber each other in an environments with subprojects when projects are in directories named identically.
    def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
    group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
    project.base.archivesName = "${project.group}-${project.name}"

    version = rootProject.ext.version

    repositories {
        // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.

        // Start with the local filesystem.
        mavenLocal()

        // ## Look into repos controlled by Airbyte.

        // This repo hosts our public artifacts and can be referenced by anyone.
        maven { url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' }

        // TODO: We might want to add a private proxy repo for maven central at some point.

        // ## Look into other, public repos.

        // Maven Central has most of everything.
        mavenCentral()

        // Jitpack is used to pull dependencies whose package names start with 'com.github.'
        maven { url 'https://jitpack.io' }
    }
}

// License generation logic.
def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
    /*
    In java, we don't have a second linter/styling tool other than spotless so it doesn't really
    matter if we write a newline or not for startComment/endComment.

    However, in python, we are using black that double-checks and reformats the code.
    Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
    big deal and would be reformatted (removed) because of black's specs.
    */
    def tmp = File.createTempFile('tmp', '.tmp')
    tmp.withWriter {
        def w = it
        if (startComment.length() > 0 || !isPython) {
            w.writeLine(startComment)
        }
        license.eachLine {
            w << lineComment
            w.writeLine(it)
        }
        if (endComment.length() > 0 || !isPython) {
            w.writeLine(endComment)
        }
        w.writeLine("")
        if (isPython) {
            w.writeLine("")
        }
    }
    return tmp
}
def createPythonLicenseWith = { license ->
    return createLicenseWith(license, '', '', '', true)
}
def createJavaLicenseWith = { license ->
    return createLicenseWith(license, '/*', ' */', ' * ', false)
}

// node is required by the root project to apply the prettier formatter repo-wide.
node {
    download = true
    version = '18.18.0'
    npmVersion = '10.1.0'
    // when setting both these directories, npm and node will be in separate directories
    workDir = file("${buildDir}/nodejs")
    npmWorkDir = file("${buildDir}/npm")
}

// python is required by the root project to apply python formatters like isort or black.
python {
    envPath = '.venv'
    minPythonVersion = '3.9'
    scope = 'VIRTUALENV'
    installVirtualenv = true
    pip 'pip:21.3.1'
    pip 'black:22.3.0'
    pip 'mypy:1.4.1'
    pip 'isort:5.6.4'
    pip 'flake8:4.0.1'
    // flake8 doesn't support pyproject.toml files
    // and thus there is the wrapper "pyproject-flake8" for this
    pip 'pyproject-flake8:0.0.1a2'
}

// format tasks per project
allprojects {

    def createFormatTarget = { pattern ->
        // We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
        // monorepo setup and it doesn't actually exclude directories reliably.
        // This code makes the behavior predictable.
        ArrayList<String> excludes = [
                '.gradle',
                'node_modules',
                '.eggs',
                '.mypy_cache',
                '.venv',
                '*.egg-info',
                'build',
                'dbt-project-template',
                'dbt-project-template-mssql',
                'dbt-project-template-mysql',
                'dbt-project-template-oracle',
                'dbt-project-template-clickhouse',
                'dbt-project-template-snowflake',
                'dbt-project-template-tidb',
                'dbt-project-template-duckdb',
                'dbt_test_config',
                'normalization_test_output',
                'tools',
                'secrets',
                'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
                'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows
                'resources/seed/*_registry.json', // Do not remove - this is also necessary to prevent diffs in our github workflows
                'resources/seed/specs_secrets_mask.yaml', // Downloaded externally.
                'resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py',
                'source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad
                'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted
                'airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', // These are deliberately invalid and unformattable.
                '__init__.py',
                'declarative_component_schema.py',
                'source-stock-ticker-api-tutorial/source.py',
                'tools/git_hooks/tests/test_spec_linter.py',
                'tools/schema_generator/schema_generator/infer_schemas.py',
        ]
        // Ensure that the excludes work when we're already partly in their path.
        excludes.addAll excludes.collectMany {
            it.startsWith("${project.name}/") ? [it, it.substring("${project.name}/".length())] : [it]
        }
        // TODO: remove this once SUB_BUILD is no longer a thing, here for github workflow backwards compatibility.
        if (System.getenv().containsKey("SUB_BUILD")) {
            excludes.add("airbyte-integrations/connectors")
        }
        // Prefix everything with double-globs because we only care about path suffixes.
        excludes = excludes.collect { "**/${it}" }
        // Remove whatever's covered by the project's subprojects, so that each file is targeted by at most one task.
        def currentProject = project
        project.subprojects {
          if (it.parent.projectDir == currentProject.projectDir) {
                excludes.add "${it.projectDir}/**"
          }
        }
        // Build the FileTree.
        return fileTree(dir: projectDir, include: pattern, exclude: excludes)
    }

    // Apply spotless formatting.
    apply plugin: 'com.diffplug.spotless'
    spotless {
        def javaTarget = createFormatTarget('**/*.java')
        if (!javaTarget.isEmpty()) {
            java {
                target javaTarget
                importOrder()
                eclipse('4.21').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))
                licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
                removeUnusedImports()
                trimTrailingWhitespace()
            }
        }
        def groovyGradleTarget = createFormatTarget('**/*.gradle')
        if (!groovyGradleTarget.isEmpty()) {
            groovyGradle {
                target groovyGradleTarget
            }
        }
        def sqlTarget = createFormatTarget('**/*.sql')
        if (!sqlTarget.isEmpty()) {
            sql {
                target sqlTarget
                dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
            }
        }
        def stylingTarget = createFormatTarget(['**/*.yaml', '**/*.yml', '**/*.json'])
        if (!stylingTarget.isEmpty()) {
            format 'styling', {
                target stylingTarget
                def npmPath = "${rootProject.tasks.named('npmSetup').get().npmDir.get()}/bin/npm"
                def nodePath = "${rootProject.tasks.named('nodeSetup').get().nodeDir.get()}/bin/node"
                prettier().npmExecutable(npmPath).nodeExecutable(nodePath)
            }
        }
    }
    tasks.matching { it.name =~ /spotless.*/ }.configureEach {
        dependsOn tasks.matching { it.name == 'generate' }
    }
    tasks.matching { it.name =~ /spotlessStyling.*/ }.configureEach {
        dependsOn rootProject.tasks.named('nodeSetup')
        dependsOn rootProject.tasks.named('npmSetup')
    }
    // TODO: remove this once the CI routinely enforces formatting checks.
    tasks.matching { it.name =~ /spotless.*Check/ }.configureEach {
        enabled = false
    }

    // python license header generation is part of 'format'
    def pythonTarget = createFormatTarget('**/*.py')
    if (!project.name.endsWith('source-stock-ticker-api-tutorial') && !pythonTarget.isEmpty()) {

        apply plugin: 'com.github.hierynomus.license'

        license {
            header rootProject.file("LICENSE_SHORT")
        }

        // Disable auto-generated java tasks, we rely on spotless there.
        tasks.matching { it.name =~ /license.*/ }.configureEach {
            enabled = false
        }

        def licensePythonCheck = tasks.register('licensePythonCheck', LicenseCheck) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = pythonTarget
            strictCheck = true
        }
        licensePythonCheck.configure {
            dependsOn tasks.matching { it.name == 'generate' }
        }
        def licensePythonFormat = tasks.register('licensePythonFormat', LicenseFormat) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = pythonTarget
            strictCheck = true
        }
        licensePythonFormat.configure {
            dependsOn tasks.matching { it.name == 'generate' }
        }

        def registerPythonFormatTask = {
            String taskName, String moduleName, String settingsFileFlagName, String... extraArgs ->
            def task = tasks.register(taskName, Exec) {
                workingDir projectDir
                commandLine rootProject.file('.venv/bin/python')
                args "-m", moduleName, settingsFileFlagName, rootProject.file('pyproject.toml').absolutePath
                args extraArgs
                pythonTarget.getFiles().forEach {
                    args projectDir.relativePath(it)
                }
            }
            task.configure {
                dependsOn rootProject.tasks.named('pipInstall')
                dependsOn tasks.matching { it.name == 'generate' }
            }
            return task
        }

        def isortPythonCheck = registerPythonFormatTask('isortPythonCheck', 'isort', '--settings-file', '--check-only')
        isortPythonCheck.configure {
            dependsOn licensePythonCheck
        }
        def isortPythonFormat = registerPythonFormatTask('isortPythonFormat', 'isort', '--settings-file')
        isortPythonFormat.configure {
            dependsOn licensePythonFormat
        }

        def blackPythonCheck = registerPythonFormatTask('blackPythonCheck', 'black', '--config', '--check')
        blackPythonCheck.configure {
            dependsOn licensePythonCheck
            dependsOn isortPythonCheck
        }
        def blackPythonFormat = registerPythonFormatTask('blackPythonFormat', 'black', '--config')
        blackPythonFormat.configure {
            dependsOn licensePythonFormat
            dependsOn isortPythonFormat
        }

        def flakePythonCheck = registerPythonFormatTask('flakePythonCheck', 'pflake8', '--config')
        flakePythonCheck.configure {
            dependsOn licensePythonCheck
            dependsOn isortPythonCheck
            dependsOn blackPythonCheck
        }
    }

    tasks.register('format') {
        dependsOn tasks.matching { it.name == 'generate' }
        dependsOn tasks.matching { it.name =~ /spotless.*Apply/ }
        dependsOn tasks.matching { it.name =~ /.*PythonFormat/ }
    }
    tasks.named('check').configure {
        dependsOn tasks.matching { it.name == 'licensePythonCheck' }
        dependsOn tasks.matching { it.name == 'isortPythonCheck' }
        dependsOn tasks.matching { it.name == 'blackPythonCheck' }
        // TODO: address flake issues then re-enable flakePythonCheck dependency
        // dependsOn tasks.matching { it.name == 'flakePythonCheck' }
    }
}

def getCDKTargetVersion() {
    def props = new Properties()
    file("airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties").withInputStream { props.load(it) }
    return props.getProperty('version')
}
static def getLatestFileModifiedTimeFromFiles(files) {
    if (files.isEmpty()) {
        return null
    }
    return files.findAll { it.isFile() }
        .collect { it.lastModified() }
        .max()
}
def checkCDKJarExists(requiredSnapshotVersion) {
    if (requiredSnapshotVersion == null) {
        // Connector does not require CDK snapshot.
        return
    }
    final boolean checkFileChanges = true
    final cdkTargetVersion = getCDKTargetVersion()
    if (requiredSnapshotVersion != cdkTargetVersion) {
        if (!cdkTargetVersion.contains("-SNAPSHOT")) {
            throw new GradleException(
                "CDK JAR version is not publishing snapshot but connector requires version ${requiredSnapshotVersion}.\n" +
                "Please check that the version in the CDK properties file matches the connector build.gradle."
            )
        }
        throw new GradleException(
            "CDK JAR version ${cdkTargetVersion} does not match connector's required version ${requiredSnapshotVersion}.\n" +
            "Please check that the version in the CDK properties file matches the connector build.gradle."
        )
    }

    def cdkJar = file("${System.properties['user.home']}/.m2/repository/io/airbyte/airbyte-cdk/${cdkTargetVersion}/airbyte-cdk-${cdkTargetVersion}.jar")
    if (!cdkJar.exists()) {
        println("WARNING: CDK JAR does not exist at ${cdkJar.path}.\nPlease run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.")
    }
    if (checkFileChanges) {
        def latestJavaFileTimestamp = getLatestFileModifiedTimeFromFiles(file("${rootDir}/airbyte-cdk/java/airbyte-cdk/src").listFiles().findAll { it.isFile() })
        if (cdkJar.lastModified() < latestJavaFileTimestamp) {
            throw new GradleException("CDK JAR is out of date. Please run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.")
        }
    }
}
static def getCDKSnapshotRequirement(dependenciesList) {
    def cdkSnapshotRequirement = dependenciesList.find {
        it.requested instanceof ModuleComponentSelector &&
        it.requested.group == 'io.airbyte' &&
        it.requested.module == 'airbyte-cdk' &&
        it.requested.version.endsWith('-SNAPSHOT')
    }
    if (cdkSnapshotRequirement == null) {
        return null
    } else {
        return cdkSnapshotRequirement.requested.version
    }
}

// Common configurations for 'assemble'.
allprojects {

    tasks.withType(Tar).configureEach {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(Zip).configureEach {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
        // Disabling distZip causes the build to break for some reason, so: instead of disabling it, make it fast.
        entryCompression ZipEntryCompression.STORED
    }
}

// Java projects common configurations.
subprojects { subproj ->

    if (!subproj.file('src/main/java').directory) {
        return
    }

    apply plugin: 'java'
    apply plugin: 'jacoco'
    apply plugin: 'com.github.spotbugs'

    java {
        sourceCompatibility = JavaVersion.VERSION_17
        targetCompatibility = JavaVersion.VERSION_17
    }


    if (isConnectorProject(subproj)) {
        // This is a Java connector project.

        // Evaluate CDK project before evaluating the connector.
        evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk')

        if (!gradle.startParameter.taskNames.any { it.contains(':airbyte-cdk:') } &&
            gradle.startParameter.taskNames.any { it.contains(':source-') || it.contains(':destination-') }) {
            // We are building a connector. Warn if the CDK JAR is missing or out of date.
            final String cdkRelativePath = 'airbyte-cdk/java/airbyte-cdk'
            afterEvaluate {
                def cdkVersionNeeded = getCDKSnapshotRequirement(configurations.compileClasspath.incoming.resolutionResult.allDependencies)
                checkCDKJarExists(cdkVersionNeeded)
            }
        }
    }

    jacoco {
        toolVersion = "0.8.8"
    }

    jacocoTestReport {
        reports {
            html.required = true
            xml.required = true
            csv.required = false
        }
    }
    def jacocoTestReportTask = tasks.named('jacocoTestReport')
    jacocoTestReportTask.configure {
        dependsOn tasks.named('test')
    }

    jacocoTestCoverageVerification {
        violationRules {
            failOnViolation = false
            rule {
                element = 'CLASS'
                excludes = ['**/*Test*', '**/generated*']
                limit {
                    counter = 'BRANCH'
                    minimum = 0.8
                }
                limit {
                    counter = 'INSTRUCTION'
                    minimum = 0.8
                }
            }
        }
    }

    spotbugs {
        ignoreFailures = false
        effort = 'max'
        excludeFilter.set rootProject.file('spotbugs-exclude-filter-file.xml')
        reportLevel = 'high'
        showProgress = false
        toolVersion = '4.7.3'
        if (rootProject.ext.skipSlowTests && isConnectorProject(subproj)) {
            effort = 'min'
        }
    }

    test {
        maxHeapSize = '3g'
        maxParallelForks = Runtime.runtime.availableProcessors()
        // This allows to set up a `gradle.properties` file inside the connector folder to reduce parallelization.
        // This is especially useful for connectors that share resources, like Redshift or Snowflake.
        if (subproj.hasProperty('numberThreads')) {
            int numberThreads = 0
            String numberThreadsString = subproj.property('numberThreads').toString()
            if (numberThreadsString.isInteger()) {
                numberThreads = numberThreadsString as int
            }
            if (numberThreads > 0 && numberThreads < maxParallelForks) {
                maxParallelForks = numberThreads
            }
        }

        if (rootProject.ext.skipSlowTests) {
            // Exclude all connector unit tests
            exclude '**/io/airbyte/integrations/source/**'
            exclude '**/io/airbyte/integrations/destination/**'
        }

        jacoco {
            enabled = !rootProject.ext.skipSlowTests
            excludes = ['**/*Test*', '**/generated*']
        }
        useJUnitPlatform {
            excludeTags('cloud-storage')
        }
        testLogging() {
            events "passed", "skipped", "failed"
            exceptionFormat 'full'
            // uncomment to get the full log output
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReportTask
    }

    // TODO: These should be added to the CDK or to the individual projects that need them:
    dependencies {
        implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0"))
        implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))

        // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
        implementation libs.bundles.jackson
        implementation libs.guava
        implementation libs.commons.io
        implementation libs.bundles.apache
        implementation libs.slf4j.api

        // SLF4J as a facade over Log4j2 required dependencies
        implementation libs.bundles.log4j

        // Bridges from other logging implementations to SLF4J
        implementation libs.bundles.slf4j

        // Dependencies for logging to cloud storage, as well as the various clients used to do so.
        implementation libs.appender.log4j2
        implementation libs.aws.java.sdk.s3
        implementation libs.google.cloud.storage

        implementation libs.s3

        // Lombok dependencies
        compileOnly libs.lombok
        annotationProcessor libs.lombok

        testCompileOnly libs.lombok
        testAnnotationProcessor libs.lombok

        testRuntimeOnly libs.junit.jupiter.engine
        testImplementation libs.bundles.junit
        testImplementation libs.assertj.core

        testImplementation libs.junit.pioneer

        // adds owasp plugin
        spotbugsPlugins libs.findsecbugs.plugin
        implementation libs.spotbugs.annotations
    }

    tasks.withType(SpotBugsTask).configureEach {
        // Reports can be found under each subproject in build/spotbugs/
        reports {
            xml.required = false
            html.required = true
        }
    }

    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
    tasks.named('check').configure {
        dependsOn tasks.named('jacocoTestCoverageVerification')
    }
}


// integration and performance test tasks per project
allprojects {
    tasks.register('integrationTest') {
        dependsOn tasks.matching {
            [
                    'integrationTestJava',
                    'customIntegrationTests', // python integration tests
                    'standardSourceTestFile',
            ].contains(it.name)
        }
    }

    tasks.register('performanceTest') {
        dependsOn tasks.matching {
            [
                    'performanceTestJava',
            ].contains(it.name)
        }
    }
}


// convenience task to list all dependencies per project
subprojects {
    tasks.register('listAllDependencies', DependencyReportTask) {}
}

// produce reproducible archives
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
tasks.withType(AbstractArchiveTask).configureEach {
    preserveFileTimestamps = false
    reproducibleFileOrder = true
}

// pin dependency versions according to ./deps.toml
catalog {
    versionCatalog {
        from(files("deps.toml"))
    }
}