Skip to content

Commit 4f3727e

Browse files
nbulsaranbulsara
authored
allow private ecr repositories as process container (#55)
* allow private ecr repositories as process container * add unit test to verify private ecr repo is recognized
1 parent 4154787 commit 4f3727e

File tree

8 files changed

+347
-2
lines changed

8 files changed

+347
-2
lines changed

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
.ipynb_checkpoints/
1+
.ipynb_checkpoints/
2+
.vscode

‎utils/scripts/nf/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,12 @@ def docker_registry(self) -> str:
150150
raise fnfe
151151
return _docker_registry
152152

153+
def is_ecr_private_repo(self, _registry_name):
154+
"""
155+
Detect private registry in format <awsaccountid>.dkr.ecr.<awsregion>.amazonaws.com
156+
"""
157+
return re.match('[0-9]{12}\.dkr\.ecr\.[a-zA-Z0-9-]{1,}\.amazonaws\.com', _registry_name)
158+
153159
def get_container_manifest(self, substitutions=None) -> list:
154160
"""
155161
generates a list of unique container image URIs to pull into an ECR Private registry
@@ -183,6 +189,9 @@ def _get_ecr_image_name(self, uri, substitutions=None, namespace_config=None):
183189
props = namespace_config.get(source_registry)
184190
if props:
185191
uri = "/".join([props['namespace'], image_name])
192+
else:
193+
if self.is_ecr_private_repo(source_registry):
194+
uri = image_name
186195

187196
return uri
188197

‎utils/scripts/tests/test_inspect_nf.py

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,29 @@
11
from glob import glob
22
from os import path
3-
3+
import json
44
import pytest
55

66
from ..nf import *
77

8+
NAMESPACE="""{
9+
"public.ecr.aws": { "namespace": "ecr-public", "pull_through": true },
10+
"ecr-public": { "namespace": "ecr-public", "pull_through": true },
11+
12+
"quay.io": { "namespace": "quay", "pull_through": true },
13+
"quay": { "namespace": "quay", "pull_through": true },
14+
15+
"us.gcr.io": { "namespace": "gcr", "pull_through": false },
16+
"eu.gcr.io": { "namespace": "gcr", "pull_through": false },
17+
"asia.gcr.io": { "namespace": "gcr", "pull_through": false },
18+
"pkg.dev": { "namespace": "gar", "pull_through": false },
19+
"nvcr.io": { "namespace": "nvcr", "pull_through": false},
20+
21+
"ghcr.io": { "namespace": "ghcr", "pull_through": false },
22+
"mcr.microsoft.com": { "namespace": "mcr", "pull_through": false },
23+
24+
"dockerhub": { "namespace": "dockerhub", "pull_through": false },
25+
"": { "namespace": "dockerhub", "pull_through": false }
26+
}"""
827

928
WORKING_DIR = path.dirname(path.realpath(__file__))
1029

@@ -105,6 +124,13 @@ def test_find_docker_uri_quoted():
105124
uri = find_docker_uri(container)
106125
assert uri == 'image:tag'
107126

127+
def test_ecr_private_repo_detection():
128+
project_path = path.join(WORKING_DIR, 'workflow_with_config_and_private_ecr')
129+
workflow = NextflowWorkflow(project_path)
130+
131+
assert "111111111111.dkr.ecr.us-east-1.amazonaws.com/rnaseq-nf:1.1.1" in workflow.containers
132+
assert workflow.is_ecr_private_repo("111111111111.dkr.ecr.us-east-1.amazonaws.com")
133+
assert not workflow.is_ecr_private_repo("1234.dcr.ecr.us-east-1.amazonaws.com")
108134

109135
def test_parse_workflow():
110136
project_path = path.join(WORKING_DIR, 'workflow')

‎utils/scripts/tests/workflow_with_config_and_private_ecr/main.nf

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
include { NO_CONTAINER } from "./processes/no_container.nf"
2+
include { FOO, BAR } from "./processes/multiple.nf"
3+
4+
workflow MAIN {
5+
NO_CONTAINER()
6+
FOO()
7+
BAR()
8+
}
9+
10+
workflow {
11+
MAIN()
12+
}

‎utils/scripts/tests/workflow_with_config_and_private_ecr/nextflow.config

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
omicsqc/omicsqc Nextflow config file
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Default config options for all compute environments
6+
----------------------------------------------------------------------------------------
7+
*/
8+
9+
// Global default params, used in configs
10+
params {
11+
12+
// TODO nf-core: Specify your pipeline's command line flags
13+
// Input options
14+
input = null
15+
16+
17+
// References
18+
genome = null
19+
igenomes_base = 's3://ngi-igenomes/igenomes'
20+
igenomes_ignore = false
21+
// MultiQC options
22+
multiqc_config = null
23+
multiqc_title = null
24+
multiqc_logo = null
25+
max_multiqc_email_size = '25.MB'
26+
multiqc_methods_description = null
27+
28+
// Boilerplate options
29+
outdir = null
30+
tracedir = "${params.outdir}/pipeline_info"
31+
publish_dir_mode = 'copy'
32+
email = null
33+
email_on_fail = null
34+
plaintext_email = false
35+
monochrome_logs = false
36+
hook_url = null
37+
help = false
38+
version = false
39+
validate_params = true
40+
show_hidden_params = false
41+
schema_ignore_params = 'genomes'
42+
43+
44+
// Config options
45+
custom_config_version = 'master'
46+
custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
47+
config_profile_description = null
48+
config_profile_contact = null
49+
config_profile_url = null
50+
config_profile_name = null
51+
52+
53+
// Max resource options
54+
// Defaults only, expecting to be overwritten
55+
max_memory = '128.GB'
56+
max_cpus = 16
57+
max_time = '240.h'
58+
59+
}
60+
61+
// Load base.config by default for all pipelines
62+
includeConfig 'conf/base.config'
63+
64+
// Load nf-core custom profiles from different Institutions
65+
try {
66+
includeConfig "${params.custom_config_base}/nfcore_custom.config"
67+
} catch (Exception e) {
68+
System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
69+
}
70+
71+
// Load omicsqc/omicsqc custom profiles from different institutions.
72+
// Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs!
73+
// try {
74+
// includeConfig "${params.custom_config_base}/pipeline/omicsqc.config"
75+
// } catch (Exception e) {
76+
// System.err.println("WARNING: Could not load nf-core/config/omicsqc profiles: ${params.custom_config_base}/pipeline/omicsqc.config")
77+
// }
78+
79+
80+
profiles {
81+
debug { process.beforeScript = 'echo $HOSTNAME' }
82+
conda {
83+
conda.enabled = true
84+
docker.enabled = false
85+
singularity.enabled = false
86+
podman.enabled = false
87+
shifter.enabled = false
88+
charliecloud.enabled = false
89+
}
90+
mamba {
91+
conda.enabled = true
92+
conda.useMamba = true
93+
docker.enabled = false
94+
singularity.enabled = false
95+
podman.enabled = false
96+
shifter.enabled = false
97+
charliecloud.enabled = false
98+
}
99+
docker {
100+
docker.enabled = true
101+
docker.userEmulation = true
102+
singularity.enabled = false
103+
podman.enabled = false
104+
shifter.enabled = false
105+
charliecloud.enabled = false
106+
}
107+
arm {
108+
docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64'
109+
}
110+
singularity {
111+
singularity.enabled = true
112+
singularity.autoMounts = true
113+
docker.enabled = false
114+
podman.enabled = false
115+
shifter.enabled = false
116+
charliecloud.enabled = false
117+
}
118+
podman {
119+
podman.enabled = true
120+
docker.enabled = false
121+
singularity.enabled = false
122+
shifter.enabled = false
123+
charliecloud.enabled = false
124+
}
125+
shifter {
126+
shifter.enabled = true
127+
docker.enabled = false
128+
singularity.enabled = false
129+
podman.enabled = false
130+
charliecloud.enabled = false
131+
}
132+
charliecloud {
133+
charliecloud.enabled = true
134+
docker.enabled = false
135+
singularity.enabled = false
136+
podman.enabled = false
137+
shifter.enabled = false
138+
}
139+
gitpod {
140+
executor.name = 'local'
141+
executor.cpus = 16
142+
executor.memory = 60.GB
143+
}
144+
test { includeConfig 'conf/test.config' }
145+
test_full { includeConfig 'conf/test_full.config' }
146+
}
147+
148+
149+
// Load igenomes.config if required
150+
if (!params.igenomes_ignore) {
151+
includeConfig 'conf/igenomes.config'
152+
} else {
153+
params.genomes = [:]
154+
}
155+
156+
157+
// Export these variables to prevent local Python/R libraries from conflicting with those in the container
158+
// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container.
159+
// See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable.
160+
161+
env {
162+
PYTHONNOUSERSITE = 1
163+
R_PROFILE_USER = "/.Rprofile"
164+
R_ENVIRON_USER = "/.Renviron"
165+
JULIA_DEPOT_PATH = "/usr/local/share/julia"
166+
}
167+
168+
// Capture exit codes from upstream processes when piping
169+
process.shell = ['/bin/bash', '-euo', 'pipefail']
170+
171+
def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
172+
timeline {
173+
enabled = true
174+
file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html"
175+
}
176+
report {
177+
enabled = true
178+
file = "${params.tracedir}/execution_report_${trace_timestamp}.html"
179+
}
180+
trace {
181+
enabled = true
182+
file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt"
183+
}
184+
dag {
185+
enabled = true
186+
file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.html"
187+
}
188+
189+
manifest {
190+
name = 'omicsqc/omicsqc'
191+
author = """Fernando Izquierdo"""
192+
homePage = 'https://github.com/omicsqc/omicsqc'
193+
description = """test workflow fastqc"""
194+
mainScript = 'main.nf'
195+
nextflowVersion = '!>=22.10.1'
196+
version = '1.0dev'
197+
doi = ''
198+
}
199+
200+
// Load modules.config for DSL2 module specific options
201+
includeConfig 'conf/modules.config'
202+
203+
// Set default registry for Docker and Podman independent of -profile
204+
// Will not be used unless Docker / Podman are enabled
205+
// Set to your registry if you have a mirror of containers
206+
//docker.registry = 'quay.io'
207+
//podman.registry = 'quay.io'
208+
209+
// Function to ensure that resource requirements don't go beyond
210+
// a maximum limit
211+
def check_max(obj, type) {
212+
if (type == 'memory') {
213+
try {
214+
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
215+
return params.max_memory as nextflow.util.MemoryUnit
216+
else
217+
return obj
218+
} catch (all) {
219+
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
220+
return obj
221+
}
222+
} else if (type == 'time') {
223+
try {
224+
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
225+
return params.max_time as nextflow.util.Duration
226+
else
227+
return obj
228+
} catch (all) {
229+
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
230+
return obj
231+
}
232+
} else if (type == 'cpus') {
233+
try {
234+
return Math.min( obj, params.max_cpus as int )
235+
} catch (all) {
236+
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
237+
return obj
238+
}
239+
}
240+
}
241+
includeConfig 'conf/omics.config'

‎utils/scripts/tests/workflow_with_config_and_private_ecr/processes/index/main.nf

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
process INDEX {
2+
container '111111111111.dkr.ecr.us-east-1.amazonaws.com/rnaseq-nf:1.1.1'
3+
4+
input:
5+
path transcriptome
6+
7+
output:
8+
path 'index'
9+
10+
script:
11+
"""
12+
echo "Running salmon index"
13+
salmon index --threads $task.cpus -t $transcriptome -i index
14+
echo "Command done"
15+
ls -lR && sleep 60
16+
"""
17+
}

‎utils/scripts/tests/workflow_with_config_and_private_ecr/processes/multiple.nf

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
process FOO {
2+
container "foo:fizz"
3+
input:
4+
val x
5+
6+
output:
7+
val y
8+
9+
script:
10+
"""
11+
echo foo
12+
"""
13+
}
14+
15+
process BAR {
16+
container "bar:buzz"
17+
input:
18+
val x
19+
20+
output:
21+
val y
22+
23+
script:
24+
"""
25+
echo bar
26+
"""
27+
}

‎utils/scripts/tests/workflow_with_config_and_private_ecr/processes/no_container.nf

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
process NO_CONTAINER {
2+
input:
3+
path x
4+
5+
output:
6+
path "out"
7+
8+
script:
9+
"""
10+
echo ${x}
11+
"""
12+
}

0 commit comments

Comments
 (0)