[Build] Simplify exclusion of artifacts during Maven publication and inline CBIaggregator script

JenkinsJobs/Releng/publishToMaven.jenkinsfile

@@ -14,60 +14,145 @@ pipeline {
 		maven 'apache-maven-latest'
 	}
 	environment {
+		// Filter out all the feature, test, and product IUs that are not published. Escape dots to match them literally
+		EXCLUDED_ARTIFACTS = "${'.feature.group$|.feature.jar$|.test|org.eclipse.equinox.executable|org.eclipse.platform.ide|org.eclipse.sdk.ide|_root$'.replace('.', '\\.')}"
 		REPO = "${WORKSPACE}/repo"
 		PATH = "${installMavenDaemon('1.0.2')}/bin:${PATH}"
 		// Folder ~/.m2 is not writable for builds, ensure mvnd metadata are written within the workspace.
 		// prevent jline warning about inability to create a system terminal and increase keep-alive timeouts to increase stability in concurrent usage
 		MVND = "mvnd -Dmvnd.daemonStorage=${WORKSPACE}/tools/mvnd -Dorg.jline.terminal.type=dumb -Dmvnd.keepAlive=1000 -Dmvnd.maxLostKeepAlive=100"
+		ECLIPSE = "${installLatestEclipse()}"
+		URL_AGG_UPDATES = 'https://download.eclipse.org/cbi/updates/p2-aggregator/products/nightly/latest'
 	}
 	// parameters declared in the definition of the invoking job
 	stages {
 		stage('Aggregate Maven repository') {
 			steps {
-				sh '''
-					SCRIPT='git-repo/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh'
-					chmod +x ${SCRIPT}
-					${SCRIPT} ${snapshotOrRelease}
-				'''
-				dir("${REPO}") {
-					sh '''#!/bin/sh -e
-						# Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings.
-						# So regenerate the sha1 for every pom.
-						for i in $(find org -name *.pom); do
-							echo "Recalculate checksum of $i"
-							sha1sum -b < $i | awk '{print $1}' > $i.sha1
+				sh '''#!/bin/bash -e
+					echo "==== Install and run the CBI aggregator ===="
+
+					DIR_AGGREGATOR="${WORKSPACE}/tools/aggregator"
+					MAVEN_PUBLISH_BASE="${WORKSPACE}/git-repo/eclipse.platform.releng/publish-to-maven-central"
+					FILE_SDK_AGGR="${MAVEN_PUBLISH_BASE}/SDK4Mvn.aggr"
+
+					# Set whether this is a snapshot build or not
+					if [ "${snapshotOrRelease}" = "-snapshot" ]; then
+						sed -e 's/snapshot=".*"/snapshot="true"/g' -i ${FILE_SDK_AGGR}
+					fi
+
+					echo "Installing the CBI aggregator into ${DIR_AGGREGATOR} ..."
+					${ECLIPSE} --launcher.suppressErrors -noSplash \\
+						-application org.eclipse.equinox.p2.director \\
+						-r ${URL_AGG_UPDATES} \\
+						-d ${DIR_AGGREGATOR} -p CBIProfile \\
+						-installIU org.eclipse.cbi.p2repo.cli.product
+
+					repoRaw="${WORKSPACE}/repo-raw"
+					mkdir ${repoRaw}
+
+					echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
+					"${DIR_AGGREGATOR}/cbiAggr" aggregate --buildModel ${FILE_SDK_AGGR} \\
+					  --action CLEAN_BUILD --buildRoot ${repoRaw} \\
+					  -vmargs -Dorg.eclipse.ecf.provider.filetransfer.excludeContributors=org.eclipse.ecf.provider.filetransfer.httpclientjava
+
+					mv ${repoRaw}/final ${REPO}
+					rm -rf ${repoRaw}
+
+					echo "==== Enrich POMs ===="
+					# Add some required information to the generated poms:
+					# - dynamic content (retrieved mostly from MANIFEST.MF):
+					#   - name
+					#   - url
+					#   - scm connection, tag and url
+					# - semi dynamic
+					#   - developers (based on static map git-repo-base -> project leads)
+					# - static content
+					#   - license
+					#   - organization
+					#   - issue management
+
+					ENRICH_POMS_JAR=${WORKSPACE}/work/EnrichPoms.jar
+					ENRICH_POMS_PACKAGE=org.eclipse.platform.releng.maven.pom
+
+					# build the jar:
+					mkdir -p ${WORKSPACE}/work/bin
+					javac -d "${WORKSPACE}/work/bin" $(find "${MAVEN_PUBLISH_BASE}/src" -name \\*.java)
+					pushd "${WORKSPACE}/work/bin"
+					jar --create --verbose --main-class=${ENRICH_POMS_PACKAGE}.EnrichPoms --file=${ENRICH_POMS_JAR} $(find * -name \\*.class)
+					popd
+					ls -l ${ENRICH_POMS_JAR}
+
+					for project in {platform,jdt,pde}; do
+					  echo "${project}"
+					  java -jar ${ENRICH_POMS_JAR} "${REPO}/org/eclipse/${project}"
+					done
+
+					echo "==== Add Javadoc stubs ===="
+
+					# (groupSimpleName, javadocArtifactGA)
+					function createJavadocs() {
+						group=${1}
+						jar="${1}-javadoc.jar"
+						artifact=${2}
+						if [ -r ${jar} ]; then
+							rm ${jar}
+						fi
+						echo -e "Corresponding javadoc can be found in artifact ${artifact}\\n" > README.txt
+						jar cf ${jar} README.txt
+						for pom in org/eclipse/${group}/*/*/*.pom; do
+							pomFolder=$(dirname ${pom})
+							if [[ ! $pomFolder =~ ${EXCLUDED_ARTIFACTS_PATTERN} ]]; then
+								javadoc=`echo ${pom} | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|"`
+								cp ${jar} ${javadoc}
+							fi
+						done	
+					}
+
+					pushd ${REPO}
+					createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
+					createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
+					createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
+
+
+					echo "==== Recalculate pom-file hashes ===="
+
+					# Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings.
+					# So regenerate the sha1 for every pom.
+					for i in $(find org -name *.pom); do
+						echo "Recalculate checksum of $i"
+						sha1sum -b < $i | awk '{print $1}' > $i.sha1
+					done
+
+					echo "========== Repo aggregation completed ========="
+
+					# Find all the artifact folders for all projects
+					for project in {platform,jdt,pde}; do
+						for pomPath in org/eclipse/${project}/*/*/*.pom; do
+							artifactId=$(basename $(dirname $(dirname ${pomPath})))
+							if [[ $artifactId =~ ${EXCLUDED_ARTIFACTS} ]]; then
+								continue # Skip excluded artifact
+							fi
+							version=$(basename $(dirname ${pomPath}))
+							groupPath=$(dirname $(dirname $(dirname ${pomPath})))
+							groupId=${groupPath//'/'/.}
+							# And transform each path to a Maven artifact coordinate groupId:artifactId:version.
+							echo "${groupId}:${artifactId}:${version}">>"${WORKSPACE}/coordinates-${project}.txt" # append the GAV
+							echo "${pomPath}">>"${WORKSPACE}/artifacts-${project}.txt"  # append the in-repo path
 						done
-					'''
-				}
+					done
+
+					popd
+				'''
 			}
 		}
 		stage('Validate repository') {
 			// It prunes down the set of artifacts to be published, e.g., eliminate test artifacts,
 			// and it tests that each to-be-published artifact can transitively resolve all its dependencies.
 			steps {
 				dir('repo-validation') { // Do the work in a clean folder without a pom.xml
-					sh '''#!/bin/bash -e
-						workingDir=$(pwd)
-						pushd "${REPO}"
-						# Find all the version folders for all projects
-						projects='org/eclipse/pde org/eclipse/jdt/ org/eclipse/platform/'
-						paths=$(find ${projects} -regextype posix-egrep -regex '.*/[0-9]+\\.[0-9]+[^/]*')
-						for path in $paths; do
-							if [[ $path =~ \\.feature\\.group|\\.feature\\.jar|\\.executable|\\.test|\\.platform\\.ide|\\.platform\\.sdk|_root|\\.id/|\\.sdk\\.ide/ ]]; then
-								# Filter out all the feature, test, and product IUs that are not published.
-								continue
-							fi
-							# And transform each path to a Maven artifact coordinate groupId:artifactId:version.
-							elements=($(echo $path | tr '/' ' ')) #split by slash
-							groupId=$(echo ${elements[@]:0:(${#elements[@]}-2)} | tr ' ' '.') # join first n-2 elements by a dot
-							gav="${groupId}:${elements[-2]}:${elements[-1]}" # 'groupId:artifactId:version'
-							echo "${gav}">>"${workingDir}/coordinates.txt"
-						done
-						popd
-
+					sh '''#!/bin/bash -xe
 						# Get each artifact and all its transitive dependencies from the Mavenized repository.
-						set -x
-						for i in $(cat coordinates.txt); do
+						for i in $(cat ${WORKSPACE}/coordinates-*.txt); do
 							${MVND} dependency:get --no-transfer-progress -Dosgi.platform=gtk.linux.x86_64 -Dartifact=$i -DremoteRepositories=file://${REPO}
 						done
 					'''
@@ -101,9 +186,9 @@ pipeline {
 									# Copy configuration pom into clean directory to stop maven from finding the .mvn folder of this git-repository
 									cp "${WORKSPACE}/git-repo/eclipse-platform-parent/pom.xml" eclipse-parent-pom.xml
 
-									for pomFile in ${REPO}/org/eclipse/${PROJECT}/*/*/*.pom; do
+									for pomFile in $(cat "${WORKSPACE}/artifacts-${PROJECT}.txt"); do
 										set +x
-										pomFolder=$(dirname ${pomFile#${REPO}/}) # name of folder, with leading REPO path stripped of
+										pomFolder=$(dirname ${pomFile})
 										version=$(basename ${pomFolder})
 										if [[ $version == *-SNAPSHOT ]]; then
 											URL=https://repo.eclipse.org/content/repositories/eclipse-snapshots/
@@ -124,6 +209,7 @@ pipeline {
 											fi
 										fi
 
+										pomFile="${REPO}/${pomFile}"
 										file=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1.jar|")
 										sourcesFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-sources.jar|")
 										javadocFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|")
@@ -170,7 +256,7 @@ pipeline {
 		always {
 			archiveArtifacts allowEmptyArchive: true, artifacts: '\
 				repo/**, \
-				repo-validation/coordinates.txt'
+				coordinates*.txt, artifacts*.txt'
 		}
 		unsuccessful {
 			emailext subject: "Publication of Maven artifacts failed",
@@ -180,10 +266,21 @@ pipeline {
 	}
 }
 
-def installMavenDaemon(String version){
+def installMavenDaemon(String version) {
 	return install('mvnd', "https://downloads.apache.org/maven/mvnd/${version}/maven-mvnd-${version}-linux-amd64.tar.gz")
 }
 
+def installLatestEclipse(){
+	def props = null
+	dir("${WORKSPACE}/git-repo") {
+		props = readProperties(file: 'cje-production/buildproperties.txt').collectEntries{n, v ->
+				v = v.trim();
+				return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)]
+			}
+	}
+	return install('eclipse', "https://download.eclipse.org/eclipse/downloads/drops4/${props.PREVIOUS_RELEASE_ID}/eclipse-SDK-${props.PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz") + '/eclipse'
+}
+
 def install(String toolType, String url) {
 	dir("${WORKSPACE}/tools/${toolType}") {
 		sh "curl -L ${url} | tar -xzf -"

RELEASE.md

@@ -108,9 +108,8 @@ The actual steps to release
     - You can subscribe to [cross-project-issues](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) to get the notifications on Simrel releases.
   * #### **Publish to Maven central**
     - Publishing to maven should happen by at least Tuesday before the release since there is up to a 24 hour delay for the maven mirrors.
-    - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) and [CBIaggregator.sh](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/CBIaggregator.sh) to the release build.
+    - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) to the release build.
       - SDK4Mvn.aggr determines what is being published to Maven
-      - Updating `CBIaggregator.sh` determines which Eclipse version is used to run the CBI aggregator
     - Run the [Publish to Maven](https://ci.eclipse.org/releng/job/Releng/job/PublishToMaven/) job in jenkins with the `-release` parameter.
     - Once that publish job has completed successfully, log into https://oss.sonatype.org/#stagingRepositories and close the Platform, JDT and PDE repositories.
       - If you do not have an account on oss.sonatype.org for performing the rest of the release request one by creating an issue like https://issues.sonatype.org/browse/OSSRH-43870 to get permissions for platform, JDT and PDE projects and tag an existing release engineer to give approval.