From 38582c4a5fadef279999cdb548d110809e73b39b Mon Sep 17 00:00:00 2001 From: Alexander Link Date: Tue, 5 May 2020 13:34:42 +0200 Subject: [PATCH 1/2] Added script to update the Jenkins plugins I added the script we use to update the plugins. The whole process should be automated. Missing: Update Jenkins to LTS via script. --- jenkinsfile-runner-base-image/README.md | 12 ++- jenkinsfile-runner-steward-image/build.sh | 4 + update.sh | 7 ++ update/.gitignore | 23 ++++ update/README.md | 50 +++++++++ update/generate.groovy | 125 ++++++++++++++++++++++ update/updatePlugins.sh | 16 +++ 7 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 update.sh create mode 100644 update/.gitignore create mode 100644 update/README.md create mode 100644 update/generate.groovy create mode 100644 update/updatePlugins.sh diff --git a/jenkinsfile-runner-base-image/README.md b/jenkinsfile-runner-base-image/README.md index a1fc74a..2fbee32 100644 --- a/jenkinsfile-runner-base-image/README.md +++ b/jenkinsfile-runner-base-image/README.md @@ -2,15 +2,19 @@ Build the image using the `./build.sh` -## Plugin versions +## Update Plugin versions -The [packager-config.yml](packager-config.yml) also contains the list of plugins to install. Creating the list manually, with groupId, artifactId, version of any plugin and the transitive dependencies, is very cumbersome. +The [packager-config.yml](packager-config.yml) also contains the list of plugins to install. Creating the list manually, with groupId, artifactId, version of any plugin and the transitive dependencies, is too cumbersome. -Below you find ways to generate this list. +To update the plugins in `packager-config.yml` run the following command (from project root): +```sh +update/updatePlugins.sh +``` ### Plugins from existing Jenkins -From the Script Console of a running Jenkins execute the following script to get the list of all installed plugins, preformatted for the `packager-config.yml`. +Although not the recommended way, you can also generate the list from an existing Jenkins instance.
+From the Script Console of this Jenkins execute the following script to get the list of all installed plugins, preformatted for the `packager-config.yml` and paste it manually. ```groovy import com.cloudbees.groovy.cps.NonCPS diff --git a/jenkinsfile-runner-steward-image/build.sh b/jenkinsfile-runner-steward-image/build.sh index ef4bb84..f17a4d1 100755 --- a/jenkinsfile-runner-steward-image/build.sh +++ b/jenkinsfile-runner-steward-image/build.sh @@ -11,6 +11,7 @@ function die() { } name=stewardci-jenkinsfile-runner +tagPrefix="${5-}" cd "$(dirname "$BASH_SOURCE")" || die @@ -18,6 +19,8 @@ git rev-parse --git-dir if [[ $? == 128 ]]; then # not in a Git checkout tag="localbuild-$(date +%y%m%d)" || die +elif [[ -n "${tagPrefix}" ]]; then + tag="${tagPrefix}_$(date +%y%m%d)_$(git log --format='%h' -n 1)" || die else tag="$(date +%y%m%d)_$(git log --format='%h' -n 1)" || die fi @@ -44,4 +47,5 @@ if [[ ${1-} == "--push" ]]; then echo "Pushing ${docker_repo}/$name:$tag" docker tag "${name}-local" "${docker_repo}/$name:$tag" || die docker push "${docker_repo}/$name:$tag" || die + echo "${docker_repo}/$name:$tag" > deployInfo.txt || die fi diff --git a/update.sh b/update.sh new file mode 100644 index 0000000..11ee785 --- /dev/null +++ b/update.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +printf "Updating plugins\n" +./update/updatePlugins.sh + +printf "\nUpdating Jenkins LTS version\n" +printf "(not automated yet - update packager-config.yml manually)\n" diff --git a/update/.gitignore b/update/.gitignore new file mode 100644 index 0000000..a1c2a23 --- /dev/null +++ b/update/.gitignore @@ -0,0 +1,23 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/update/README.md b/update/README.md new file mode 100644 index 0000000..002cee5 --- /dev/null +++ b/update/README.md @@ -0,0 +1,50 @@ +# Update Plugin List + +To update the plugin versions use the `generate.groovy` script to generate an updated list of plugins with dependencies based on the latest Update Site. + +Usage: +```groovy +Usage: groovy generate.groovy [--skip-optional] + pluginsListFile Path to a file with plugin names (new line separated) of wanted plugins + outputFormat -cwp Custom War Packager packager-config.yml format (default) + -list Simple list of plugin names + -tree Print the dependency tree of each plugin + --skip-optional Do not include optional plugin dependencies +``` + +e.g. `groovy generate.groovy "../jenkinsfile-runner-base-image/plugins.txt" -cwp --skip-optional` + +will produce: +```yaml +# AnsiColor (Jenkins-Version: 2.145) + - groupId: "org.jenkins-ci.plugins" + artifactId: "ansicolor" + source: + version: "0.6.2" +# Pipeline: API (Jenkins-Version: 2.138.4) + - groupId: "org.jenkins-ci.plugins.workflow" + artifactId: "workflow-api" + source: + version: "2.34" +# Pipeline: Step API (Jenkins-Version: 2.121.1) + - groupId: "org.jenkins-ci.plugins.workflow" + artifactId: "workflow-step-api" + source: + version: "2.19" +# Structs (Jenkins-Version: 2.60.3) + - groupId: "org.jenkins-ci.plugins" + artifactId: "structs" + source: + version: "1.19" +[...] +``` + +This can be copied/pasted into the `plugins` section of a [Custom War Packager] `packager-config.yml`. Or better run the `update.sh` in the project root folder. + +To print the dependency tree run e.g. +```sh +groovy generate.groovy "../jenkinsfile-runner-base-image/plugins.txt" -tree --skip-optional +``` + + +[Custom War Packager]: https://github.com/jenkinsci/custom-war-packager \ No newline at end of file diff --git a/update/generate.groovy b/update/generate.groovy new file mode 100644 index 0000000..963ebd0 --- /dev/null +++ b/update/generate.groovy @@ -0,0 +1,125 @@ +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +wantedPlugins = [:] +resultingPlugins = [:] +updateCenter = null +debug = false + +if(args.length == 1) { + process(args[0], "-cwp") +} else if(args.length >= 2) { + if(args[1] == "-cwp" || args[1] == "-list" || args[1] == "-tree") { + skipOptional = (args.length == 3 && args[2] == "--skip-optional") + process(args[0], args[1], skipOptional) + } else { + println("Unknown argument " + args[1]) + } +} else { + println "Usage: groovy generate.groovy [--skip-optional]" + println " pluginsListFile Path to a file with plugin names (new line separated) of wanted plugins" + println " outputFormat -cwp Custom War Packager packager-config.yml format (default)" + println " -list Simple list of plugin names" + println " -tree Print the dependency tree of each plugin" + println " --skip-optional Do not include optional plugin dependencies" + println "" + println "e.g. groovy generate.groovy plugins.txt -cwp" +} + +def process(wantedPluginsFile, outFormat, skipOptional) { + def wantedPluginNames = [] + String fileContents = new File(wantedPluginsFile).getText('UTF-8').eachLine { line -> + wantedPluginNames << line + } + + def url = "https://updates.jenkins.io/update-center.json".toURL() + def updateCenterJson = url.text + updateCenterJson = updateCenterJson.substring(0, updateCenterJson.length()-2) + updateCenterJson = updateCenterJson.replace("updateCenter.post(", "") + updateCenterJson = updateCenterJson.trim() + + def jsonSlurper = new JsonSlurper() + updateCenter = jsonSlurper.parseText(updateCenterJson) + + for(wanted in wantedPluginNames){ + def wantedPlugin = updateCenter.plugins[wanted] + wantedPlugins[wantedPlugin.name] = wantedPlugin + resultingPlugins[wantedPlugin.name] = wantedPlugin + if(debug) println "Added: " + wantedPlugin.name + addDependencies(wantedPlugin, skipOptional) + } + + if(outFormat == "-list") { + for(plugin in resultingPlugins.values()){ + println plugin.name + } + } + + if(outFormat == "-cwp") { + for(pluginKey in resultingPlugins.sort()*.key){ + plugin = resultingPlugins[pluginKey] + def gav = plugin.gav + def matcher = gav =~ /([^:]*):([^:]*):([^:]*)/ + if (!matcher) throw new RuntimeException("Wrong gav: " + gav) + def groupId = matcher.group(1) + def artifactId = matcher.group(2) + def version = matcher.group(3) + println "# " + plugin.title + " (Jenkins-Version: " + plugin.requiredCore + ")" + println " - groupId: \"" + groupId + "\"" + println " artifactId: \"" + artifactId + "\"" + println " source:" + println " version: \"" + version + "\"" + } + } + + if(outFormat == "-tree") { + for(pluginKey in wantedPlugins.sort()*.key){ + plugin = wantedPlugins[pluginKey] + printDependencyTree(plugin, 0, "wanted") + } + println("--- Statistics -------------------------") + println("Wanted plugins: " + wantedPlugins.size()) + println("Resulting plugins: " + resultingPlugins.size()) + } + +} + +def addDependencies(plugin, skipOptional) { + //println "addDependencies(" + plugin + ")" + for(dependency in plugin.dependencies){ + if(skipOptional && dependency.optional) { + continue; + } + def dependencyPlugin = updateCenter.plugins[dependency.name] + if(dependencyPlugin) { + resultingPlugins[dependency.name] = dependencyPlugin + if(debug) println "Added dependency (of " + plugin.name + "): " + dependencyPlugin.name + addDependencies(dependencyPlugin, skipOptional) + } else { + if(dependency.optional) { + println "ERROR: Could not find (optional) dependency " + dependency + } else { + throw new RuntimeException("Could not find (required) dependency " + dependency) + } + } + } +} + +def printDependencyTree(plugin, indent, type) { + if(indent == 0) { + print "- " + } else { + for(i = 0; i < indent; i++) { + print " " + } + } + println plugin.name + ":" + plugin.version + " (" + type + ")" + for(dependency in plugin.dependencies){ + dependencyPlugin = resultingPlugins[dependency.name] + if(dependencyPlugin == null) { + if(!dependency.optional) throw new RuntimeException(dependency.name + " (required) not contained in resultingPlugins") + } else { + printDependencyTree(dependencyPlugin, indent+1, dependency.optional ? "optional" : "required") + } + } +} diff --git a/update/updatePlugins.sh b/update/updatePlugins.sh new file mode 100644 index 0000000..e0ce170 --- /dev/null +++ b/update/updatePlugins.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PROJECT_ROOT=$(cd "$(dirname "$BASH_SOURCE")/.."; pwd) +packagerConfig="${PROJECT_ROOT}/jenkinsfile-runner-base-image/packager-config.yml" +pluginList="${PROJECT_ROOT}/jenkinsfile-runner-base-image/plugins.txt" + +# Remove outdated plugins list +sed -e "/^##PLUGINS-START/,/^##PLUGINS-END/d" "${packagerConfig}" > "${packagerConfig}.tmp" +mv "${packagerConfig}.tmp" "${packagerConfig}" + +# Generate new plugins list +echo "##PLUGINS-START (do not remove!)" >> "${packagerConfig}" +groovy "${PROJECT_ROOT}/update/generate.groovy" "${pluginList}" -cwp --skip-optional >> "${packagerConfig}" +echo "##PLUGINS-END (do not remove!)" >> "${packagerConfig}" + +echo "Updated plugins in ${packagerConfig}" \ No newline at end of file From 4d0d6b055624d778295e2d8fac6af7bdade88c64 Mon Sep 17 00:00:00 2001 From: Alexander Link <33052602+alxsap@users.noreply.github.com> Date: Wed, 6 May 2020 09:39:51 +0200 Subject: [PATCH 2/2] missing new line --- update/updatePlugins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update/updatePlugins.sh b/update/updatePlugins.sh index e0ce170..433fb9d 100644 --- a/update/updatePlugins.sh +++ b/update/updatePlugins.sh @@ -13,4 +13,4 @@ echo "##PLUGINS-START (do not remove!)" >> "${packagerConfig}" groovy "${PROJECT_ROOT}/update/generate.groovy" "${pluginList}" -cwp --skip-optional >> "${packagerConfig}" echo "##PLUGINS-END (do not remove!)" >> "${packagerConfig}" -echo "Updated plugins in ${packagerConfig}" \ No newline at end of file +echo "Updated plugins in ${packagerConfig}"