Skip to content

Support downloading plugins using ORAS #5968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions modules/nf-commons/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ dependencies {
api 'dev.failsafe:failsafe:3.1.0'
// patch gson dependency required by pf4j
api 'com.google.code.gson:gson:2.10.1'
implementation 'land.oras:oras-java-sdk:0.2.4'

/* testImplementation inherited from top gradle build file */
testImplementation(testFixtures(project(":nextflow")))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package nextflow.plugin

import groovy.transform.CompileStatic
import land.oras.ContainerRef
import land.oras.Manifest
import land.oras.Registry
import org.pf4j.PluginRuntimeException

import java.nio.file.Files
import java.nio.file.Path

/**
* Support downloading plugins from an ORAS (OCI Registry As Storage)
* location.
*
* NOTE: logically this should implement pf4j's FileDownloader interface,
* but unfortunately that interface uses java.net.URL which doesn't support
* custom URI protocols.
*
* See https://oras.land/
*/
@CompileStatic
class OrasPluginDownloader {
private static final String PROTOCOL = "oras://"
private static final String ARTIFACT_TYPE = "application/vnd.nextflow.plugin+zip"

static boolean canDownload(String uri) {
return uri.startsWith(PROTOCOL)
}

Path downloadFile(String uri) throws IOException {
if (!canDownload(uri)) {
throw new PluginRuntimeException("URI protocol not supported by ORAS downloader: {}", uri);
}
return downloadOras(uri)
}

private static Path downloadOras(String uri) {
Path destination = Files.createTempDirectory("oras-update-downloader")
destination.toFile().deleteOnExit()

// strip the oras:// prefix since it's just a marker
ContainerRef source = ContainerRef.parse(uri.replaceFirst("oras://", ""))
Registry registry = Registry.Builder.builder()
// use http on localhost, require https everywhere else
.withInsecure(source.registry.startsWith("localhost:"))
.build()

// grab oras metadata about the url
Manifest manifest = registry.getManifest(source)
String type = manifest.getArtifactType()
if (type != ARTIFACT_TYPE) {
throw new PluginRuntimeException("Not a nextflow plugin: {}", uri)
}
// get the filename of the plugin artifact, which should be in layer 0
def layers = manifest.getLayers()
if (layers.size() < 1) {
throw new PluginRuntimeException("Unable to find nextflow plugin at: {}", uri)
}
String filename = layers[0].annotations['org.opencontainers.image.title']

// download the plugin artifact
registry.pullArtifact(source, destination, true)
return destination.resolve(filename)
}
}
23 changes: 23 additions & 0 deletions modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package nextflow.plugin

import org.pf4j.update.FileVerifier

import static java.nio.file.StandardCopyOption.*

import java.nio.file.Files
Expand Down Expand Up @@ -244,6 +246,27 @@ class PluginUpdater extends UpdateManager {
return Failsafe.with(policy).get(supplier)
}

// override this method from the parent class to support oras:// downloads
@Override
protected Path downloadPlugin(String id, String version) {
try {
def release = findReleaseForPlugin(id, version)

// pf4j's FileDownloader interface uses java.net.URL, which
// doesn't support custom protocols. This means we can't use
// it for oras:// URIs and instead have to call the ORAS downloader
// directly in such cases
Path downloaded = OrasPluginDownloader.canDownload(release.url)
? new OrasPluginDownloader().downloadFile(release.url)
: getFileDownloader(id).downloadFile(new URL(release.url))

getFileVerifier(id).verify(new FileVerifier.Context(id, release), downloaded);
return downloaded;
} catch (IOException e) {
throw new PluginRuntimeException(e, "Error during download of plugin {}", id);
}
}

protected <T> RetryPolicy<T> retryPolicy(String id, String version) {
final listener = new dev.failsafe.event.EventListener<ExecutionAttemptedEvent<T>>() {
@Override
Expand Down