From b6eacfea7d4bee628daae7a10add91535ecdf9b9 Mon Sep 17 00:00:00 2001 From: Rodrigo Antunes Date: Wed, 22 May 2024 14:57:04 -0300 Subject: [PATCH] Optaplanner weekly Jenkins job --- .ci/jenkins/Jenkinsfile.weekly.deploy | 323 +++++++++++++++++++ .ci/jenkins/dsl/jobs.groovy | 54 ++++ .ci/jenkins/project/Jenkinsfile.setup-branch | 18 ++ .ci/jenkins/project/Jenkinsfile.weekly | 188 +++++++++++ 4 files changed, 583 insertions(+) create mode 100644 .ci/jenkins/Jenkinsfile.weekly.deploy create mode 100644 .ci/jenkins/project/Jenkinsfile.weekly diff --git a/.ci/jenkins/Jenkinsfile.weekly.deploy b/.ci/jenkins/Jenkinsfile.weekly.deploy new file mode 100644 index 00000000000..de68f0f1784 --- /dev/null +++ b/.ci/jenkins/Jenkinsfile.weekly.deploy @@ -0,0 +1,323 @@ +/* groovylint-disable UnnecessaryGetter */ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.jenkinsci.plugins.workflow.libs.Library + +@Library('jenkins-pipeline-shared-libraries') _ + +import org.kie.jenkins.MavenCommand + +deployProperties = [:] + +optaplannerRepository = 'incubator-kie-optaplanner' +optaplannerFolder = 'optaplanner' +quickstartsRepository = 'incubator-kie-optaplanner-quickstarts' +quickstartsFolder = 'quickstarts' + +imageUtils = null + +pipeline { + agent { + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + label util.avoidFaultyNodes() + } + } + + options { + timestamps() + timeout(time: 120, unit: 'MINUTES') + disableConcurrentBuilds(abortPrevious: true) + } + + environment { + OPTAPLANNER_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") + } + + stages { + stage('Initialize') { + steps { + script { + cleanWs(disableDeferredWipeout: true) + + if (params.DISPLAY_NAME) { + currentBuild.displayName = params.DISPLAY_NAME + } + + checkout scm // To make sure the repository containing the script is available on the Jenkins node. + imageUtils = load '.ci/jenkins/scripts/imageUtils.groovy' + + env.PROJECT_VERSION = maven.mvnGetVersionProperty(getMavenCommand(), 'project.version') + } + } + post { + success { + script { + setDeployPropertyIfNeeded('git.branch', getBuildBranch()) + setDeployPropertyIfNeeded('git.branchQuickstarts', getQuickStartsBranch()) + setDeployPropertyIfNeeded('git.author', getGitAuthor()) + setDeployPropertyIfNeeded('project.version', getProjectVersion()) + } + } + } + } + + stage('Clone repositories') { + steps { + script { + checkoutRepo(optaplannerRepository, optaplannerFolder) + checkoutRepo(quickstartsRepository, quickstartsFolder) + } + } + } + + stage('Update project version') { + steps { + script { + if (getDroolsVersion()) { + maven.mvnSetVersionProperty(getOptaplannerMavenCommand(), 'version.org.drools', getDroolsVersion()) + } + maven.mvnVersionsSet(getOptaplannerMavenCommand(), getProjectVersion(), true) + mavenCleanInstallOptaPlannerParents() + updateQuickstartsVersions() + } + } + } + + stage('Build OptaPlanner') { + steps { + script { + withCredentials([usernamePassword(credentialsId: env.MAVEN_REPO_CREDS_ID, usernameVariable: 'REPOSITORY_USER', passwordVariable: 'REPOSITORY_TOKEN')]) { + def installOrDeploy + if (shouldDeployToRepository()) { + installOrDeploy = "deploy -DdeployAtEnd -Dapache.repository.username=${REPOSITORY_USER} -Dapache.repository.password=${REPOSITORY_TOKEN} -DretryFailedDeploymentCount=5" + } else { + installOrDeploy = 'install' + } + configFileProvider([configFile(fileId: env.MAVEN_SETTINGS_CONFIG_FILE_ID, variable: 'MAVEN_SETTINGS_FILE')]) { + getOptaplannerMavenCommand() + .withProperty('maven.test.failure.ignore', true) + .withProperty('operator.image.build') + .skipTests(params.SKIP_TESTS) + .withSettingsXmlFile(MAVEN_SETTINGS_FILE) + .run("clean $installOrDeploy") + } + } + } + } + post { + always { + script { + archiveJUnitTestResults() + util.archiveConsoleLog() + } + } + } + } + + stage('Build Quickstarts') { + steps { + script { + configFileProvider([configFile(fileId: env.MAVEN_SETTINGS_CONFIG_FILE_ID, variable: 'MAVEN_SETTINGS_FILE')]) { + getOptaplannerQuickstartsMavenCommand() + .withProperty('maven.test.failure.ignore', true) + .skipTests(params.SKIP_TESTS) + .withSettingsXmlFile(MAVEN_SETTINGS_FILE) + .run('clean install') + } + } + } + post { + always { + script { + archiveJUnitTestResults() + util.archiveConsoleLog() + } + } + } + } + + stage('Create and push a new tag') { + steps { + script { + projectVersion = getProjectVersion(false) + dir(optaplannerFolder) { + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) + githubscm.tagRepository(projectVersion) + githubscm.pushRemoteTag('origin', projectVersion, getGitAuthorPushCredsId()) + } + } + } + } + } + post { + always { + script { + def propertiesStr = deployProperties.collect { entry -> "${entry.key}=${entry.value}" }.join('\n') + writeFile(text: propertiesStr, file: env.PROPERTIES_FILE_NAME) + archiveArtifacts(artifacts: env.PROPERTIES_FILE_NAME) + } + } + unsuccessful { + sendErrorNotification() + } + cleanup { + script { + util.cleanNode() + } + } + } +} + +void sendErrorNotification() { + if (params.SEND_NOTIFICATION) { + String additionalInfo = "**[${getBuildBranch()}] Optaplanner - Deploy**" + mailer.sendMarkdownTestSummaryNotification('CI failures', [env.OPTAPLANNER_CI_EMAIL_TO], additionalInfo) + } else { + echo 'No notification sent per configuration' + } +} + +void updateQuickstartsVersions() { + maven.mvnSetVersionProperty(getOptaplannerQuickstartsMavenCommand(), 'version.org.optaplanner', getProjectVersion()) + maven.mvnVersionsUpdateParentAndChildModules(getOptaplannerQuickstartsMavenCommand(), getProjectVersion(), true) + gradleVersionsUpdate(quickstartsFolder, getProjectVersion()) +} + +void gradleVersionsUpdate(String folder, String newVersion) { + dir(folder) { + sh "find . -name build.gradle -exec sed -i -E 's/def optaplannerVersion = \"[^\"\\s]+\"/def optaplannerVersion = \"${newVersion}\"/' {} \\;" + } +} + +void archiveJUnitTestResults() { + if (!skipUnitTests()) { + junit testResults: '**/target/surefire-reports/**/*.xml, **/target/failsafe-reports/**/*.xml', allowEmptyResults: true + } +} + +void checkoutRepo(String repo, String dirName) { + dir(dirName) { + deleteDir() + checkout(githubscm.resolveRepository(repo, getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsId())) + // need to manually checkout branch since on a detached branch after checkout command + sh "git checkout ${getBuildBranch()}" + checkoutDatetime = getCheckoutDatetime() + if (checkoutDatetime) { + sh "git checkout `git rev-list -n 1 --before=\"${checkoutDatetime}\" ${getBuildBranch()}`" + } + } +} + +MavenCommand getMavenDefaultCommand() { + MavenCommand mvnCmd = new MavenCommand(this, ['-fae', '-ntp']) + if (env.MAVEN_DEPENDENCIES_REPOSITORY) { + mvnCmd.withDependencyRepositoryInSettings('deps-repo', env.MAVEN_DEPENDENCIES_REPOSITORY) + } + return mvnCmd +} + +MavenCommand getOptaplannerMavenCommand() { + return getMavenDefaultCommand().inDirectory(optaplannerFolder).withProperty('full') +} + +MavenCommand getOptaplannerQuickstartsMavenCommand() { + return getMavenDefaultCommand().inDirectory(quickstartsFolder).withProperty('full') +} + +/** + * Builds the parent modules and the BOM so that project depending on these artifacts can resolve. + */ +void mavenCleanInstallOptaPlannerParents() { + configFileProvider([configFile(fileId: env.MAVEN_SETTINGS_CONFIG_FILE_ID, variable: 'MAVEN_SETTINGS_FILE')]) { + getOptaplannerMavenCommand() + .skipTests(true) + .withOptions(['-U', '-pl org.optaplanner:optaplanner-build-parent,org.optaplanner:optaplanner-bom', '-am']) + .withSettingsXmlFile(MAVEN_SETTINGS_FILE) + .run('clean install') + } +} + +// Getters and Setters of params/properties + +boolean shouldDeployToRepository() { + return (env.MAVEN_DEPLOY_REPOSITORY || isNotTestingBuild()) && !isDeployDisabled() +} + +boolean isNotTestingBuild() { + return getGitAuthor() == 'apache' +} + +boolean skipUnitTests() { + return params.SKIP_TESTS +} + +String getGitAuthor() { + // GIT_AUTHOR can be env or param + return "${GIT_AUTHOR}" +} + +String getGitAuthorCredsId() { + return env.GIT_AUTHOR_CREDS_ID +} + +String getGitAuthorPushCredsId() { + return env.GIT_AUTHOR_PUSH_CREDS_ID +} + +String getBuildBranch() { + return params.BUILD_BRANCH_NAME +} + +String getDroolsVersion() { + return getProjectVersion() +} + +void setDeployPropertyIfNeeded(String key, def value) { + if (value != null && value != '') { + deployProperties[key] = value + } +} + +String getQuickStartsBranch() { + return params.QUICKSTARTS_BUILD_BRANCH_NAME +} + +boolean isDeployDisabled() { + return env.DISABLE_DEPLOY.toBoolean() +} + +String getCheckoutDatetime() { + return params.GIT_CHECKOUT_DATETIME +} + +String getProjectVersionDate() { + def projectVersionDate = (getCheckoutDatetime() =~ /(\d{4}-\d{2}-\d{2})/)[0][0] + return projectVersionDate.replace('-', '') +} + +String getProjectVersion(boolean keepSnapshotSuffix = true) { + def projectVersion = env.PROJECT_VERSION + if (keepSnapshotSuffix) { + return projectVersion.replace('-SNAPSHOT', "-${getProjectVersionDate()}-SNAPSHOT") + } + return projectVersion.replace('-SNAPSHOT', "-${getProjectVersionDate()}") +} diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index 37622bfc1a4..6d0468aafd2 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -49,6 +49,9 @@ createProjectSetupBranchJob() // Nightly jobs setupProjectNightlyJob() +// Weekly jobs +setupProjectWeeklyJob() + // Release jobs setupProjectReleaseJob() setupProjectPostReleaseJob() @@ -124,6 +127,21 @@ void setupProjectNightlyJob() { } } +void setupProjectWeeklyJob() { + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-weekly', JobType.OTHER, "${jenkins_path_project}/Jenkinsfile.weekly", 'Optaplanner Weekly') + jobParams.triggers = [cron : '0 5 * * 0'] + jobParams.env.putAll([ + JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", + + GIT_BRANCH_NAME: "${GIT_BRANCH}", + ]) + KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { + parameters { + booleanParam('SKIP_TESTS', false, 'Skip all tests') + } + } +} + void setupProjectReleaseJob() { def jobParams = JobParamsUtils.getBasicJobParams(this, 'optaplanner-release', JobType.RELEASE, "${jenkins_path_project}/Jenkinsfile.release", 'Optaplanner Release') jobParams.env.putAll([ @@ -227,6 +245,9 @@ setupSpecificBuildChainNightlyJob('native') setupDeployJob(JobType.RELEASE) setupPromoteJob(JobType.RELEASE) +// Weekly deploy job +setupWeeklyDeployJob() + if (Utils.isMainBranch(this)) { setupOptaPlannerTurtleTestsJob('drools') setupOptaPlannerTurtleTestsJob('bavet') @@ -408,3 +429,36 @@ void setupOptaPlannerTurtleTestsJob(String constraintStreamImplType) { } } } + +void setupWeeklyDeployJob() { + def jobParams = JobParamsUtils.getBasicJobParams(this, 'optaplanner.weekly-deploy', JobType.OTHER, "${jenkins_path}/Jenkinsfile.weekly.deploy", 'Optaplanner Weekly Deploy') + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) + jobParams.env.putAll([ + PROPERTIES_FILE_NAME: 'deployment.properties', + JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", + + GIT_AUTHOR: "${GIT_AUTHOR_NAME}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", + + MAVEN_SETTINGS_CONFIG_FILE_ID: "${MAVEN_SETTINGS_FILE_ID}", + MAVEN_DEPENDENCIES_REPOSITORY: "${MAVEN_ARTIFACTS_REPOSITORY}", + MAVEN_DEPLOY_REPOSITORY: "${MAVEN_ARTIFACTS_UPLOAD_REPOSITORY_URL}", + MAVEN_REPO_CREDS_ID: "${MAVEN_ARTIFACTS_UPLOAD_REPOSITORY_CREDS_ID}", + ]) + KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { + parameters { + stringParam('DISPLAY_NAME', '', 'Setup a specific build display name') + + stringParam('BUILD_BRANCH_NAME', "${GIT_BRANCH}", 'Set the Git branch to checkout') + + booleanParam('SKIP_TESTS', false, 'Skip tests') + + stringParam('QUICKSTARTS_BUILD_BRANCH_NAME', Utils.isMainBranch(this) ? 'development' : "${GIT_BRANCH}", 'Base branch for quickstarts.') + + stringParam('GIT_CHECKOUT_DATETIME', '', 'Git checkout date and time - (Y-m-d H:i)') + + booleanParam('SEND_NOTIFICATION', false, 'In case you want the pipeline to send a notification on CI channel for this run.') + } + } +} diff --git a/.ci/jenkins/project/Jenkinsfile.setup-branch b/.ci/jenkins/project/Jenkinsfile.setup-branch index f93412ebe4d..78c7657c06c 100644 --- a/.ci/jenkins/project/Jenkinsfile.setup-branch +++ b/.ci/jenkins/project/Jenkinsfile.setup-branch @@ -108,6 +108,24 @@ pipeline { } } + // Launch the weekly to deploy all artifacts from the branch + stage('Launch the weekly') { + when { + expression { return params.DEPLOY } + } + steps { + script { + def buildParams = getDefaultBuildParams() + addBooleanParam(buildParams, 'SKIP_TESTS', true) + buildJob('../other/optaplanner.weekly-deploy', buildParams, 'optaplanner.weekly-deploy', false) + } + } + post { + failure { + addFailedStage('optaplanner-weekly') + } + } + } } post { unsuccessful { diff --git a/.ci/jenkins/project/Jenkinsfile.weekly b/.ci/jenkins/project/Jenkinsfile.weekly new file mode 100644 index 00000000000..89fb97b3e99 --- /dev/null +++ b/.ci/jenkins/project/Jenkinsfile.weekly @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.jenkinsci.plugins.workflow.libs.Library + +@Library('jenkins-pipeline-shared-libraries')_ + +// Deploy jobs +OPTAPLANNER_DEPLOY = 'optaplanner.weekly-deploy' + +// Map of executed jobs +// See https://javadoc.jenkins.io/plugin/workflow-support/org/jenkinsci/plugins/workflow/support/steps/build/RunWrapper.html +// for more options on built job entity +JOBS = [:] + +FAILED_STAGES = [:] +UNSTABLE_STAGES = [:] + +// Should be multibranch pipeline +pipeline { + agent { + label util.avoidFaultyNodes('ubuntu') + } + + options { + timeout(time: 1380, unit: 'MINUTES') + } + + environment { + OPTAPLANNER_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") + + // Use branch name in weekly tag as we may have parallel main and release branch builds + WEEKLY_TAG = """${getBuildBranch()}-${sh( + returnStdout: true, + script: 'date -u "+%Y-%m-%d"' + ).trim()}""" +} + + stages { + stage('Initialize') { + steps { + script { + echo "weekly tag is ${env.WEEKLY_TAG}" + + currentBuild.displayName = env.WEEKLY_TAG + } + } + } + + stage('Build & Deploy OptaPlanner') { + steps { + script { + def buildParams = getDefaultBuildParams() + addSkipTestsParam(buildParams) + addSkipIntegrationTestsParam(buildParams) + + buildJob(OPTAPLANNER_DEPLOY, buildParams) + } + } + post { + failure { + addFailedStage(OPTAPLANNER_DEPLOY) + } + } + } + } + post { + unsuccessful { + script { + if (currentBuild.currentResult != 'UNSTABLE') { + sendPipelineErrorNotification() + } + } + } + } +} + +def buildJob(String jobName, List buildParams, String jobKey = jobName) { + echo "[${jobKey}] Build ${jobName} with params ${buildParams}" + + def job = build(job: "${jobName}", wait: true, parameters: buildParams, propagate: false) + JOBS[jobKey] = job + + // Set Unstable if job did not succeed + if (!isJobSucceeded(jobKey)) { + addUnstableStage(jobKey) + unstable("Job ${jobName} finished with result ${job.getResult()}") + } + return job +} + +def getJob(String jobKey) { + return JOBS[jobKey] +} + +String getJobUrl(String jobKey) { + echo "getJobUrl for ${jobKey}" + return getJob(jobKey)?.getAbsoluteUrl() ?: '' +} + +boolean isJobSucceeded(String jobKey) { + return getJob(jobKey)?.getResult() == 'SUCCESS' +} + +void addFailedStage(String jobKey = '') { + FAILED_STAGES.put("${STAGE_NAME}", jobKey) +} +void addUnstableStage(String jobKey = '') { + UNSTABLE_STAGES.put("${STAGE_NAME}", jobKey) +} + +void sendPipelineErrorNotification() { + String bodyMsg = "Optaplanner weekly job #${BUILD_NUMBER} was: ${currentBuild.currentResult}" + + paramsStr = '' + if (params.SKIP_TESTS) { + paramsStr += '\n- Tests skipped' + } + bodyMsg += paramsStr ? "\n\nConfiguration:${paramsStr}" : '\n' + + if (FAILED_STAGES.size() > 0) { + bodyMsg += '\nFailed stages: \n- ' + bodyMsg += FAILED_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ') + } + bodyMsg += '\n' + if (UNSTABLE_STAGES.size() > 0) { + bodyMsg += '\nUnstable stages: \n- ' + bodyMsg += UNSTABLE_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ') + } + bodyMsg += '\n' + bodyMsg += "\nPlease look here: ${BUILD_URL}" + emailext body: bodyMsg, subject: "[${getBuildBranch()}][d] Full Pipeline", + to: env.OPTAPLANNER_CI_EMAIL_TO +} + +List getDefaultBuildParams() { + List params = [] + addStringParam(params, 'DISPLAY_NAME', "${env.WEEKLY_TAG}") + addStringParam(params, 'GIT_CHECKOUT_DATETIME', getCheckoutDatetime()) + addStringParam(params, 'QUICKSTARTS_BUILD_BRANCH_NAME', getBuildBranch()) + addBooleanParam(params, 'SEND_NOTIFICATION', true) + + return params +} + +void addSkipTestsParam(buildParams) { + addBooleanParam(buildParams, 'SKIP_TESTS', params.SKIP_TESTS) +} + +void addSkipIntegrationTestsParam(buildParams) { + addBooleanParam(buildParams, 'SKIP_INTEGRATION_TESTS', params.SKIP_TESTS) +} + +void addStringParam(List buildParams, String key, String value) { + buildParams.add(string(name: key, value: value)) +} + +void addBooleanParam(List buildParams, String key, boolean value) { + buildParams.add(booleanParam(name: key, value: value)) +} + +String getBuildBranch() { + return env.GIT_BRANCH_NAME +} + +String getCurrentDate() { + return sh(returnStdout: true, script: 'date -u "+%Y-%m-%d"').trim() +} + +String getCheckoutDatetime() { + return getCurrentDate() + ' 02:00' // Cut-off 02:00AM +}