Merge remote-tracking branch 'origin/test' into prod

pgap.cwl

@@ -47,6 +47,9 @@ outputs:
   gff:
     outputSource: standard_pgap/gff
     type: File
+  gff_enhanced:
+    outputSource: standard_pgap/gff_enhanced
+    type: File
   sqn:
     outputSource: standard_pgap/sqn
     type: File
@@ -62,6 +65,12 @@ outputs:
   protein_fasta:
     outputSource:  standard_pgap/protein_fasta
     type: File?
+  cds_nucleotide_fasta:
+    outputSource: standard_pgap/cds_nucleotide_fasta
+    type: File?
+  cds_protein_fasta:
+    outputSource:  standard_pgap/cds_protein_fasta
+    type: File?
   initial_asndisc_error_diag:
     type: File?
     outputSource: standard_pgap/initial_asndisc_error_diag 
@@ -171,5 +180,5 @@ steps:
       no_internet: no_internet
       make_uuid: make_uuid
       uuid_in: uuid_in
-    out: [gbent, gbk, gff, nucleotide_fasta, protein_fasta, sqn, initial_asndisc_error_diag, initial_asnval_error_diag, final_asndisc_error_diag, final_asnval_error_diag, checkm_raw, checkm_results]
+    out: [gbent, gbk, gff, gff_enhanced, nucleotide_fasta, protein_fasta, cds_nucleotide_fasta, cds_protein_fasta, sqn, initial_asndisc_error_diag, initial_asnval_error_diag, final_asndisc_error_diag, final_asnval_error_diag, checkm_raw, checkm_results]
     run: wf_common.cwl

progs/asn2fasta.cwl

@@ -13,6 +13,10 @@ inputs:
     type: string
     inputBinding:
         prefix: -type 
+  feats:
+    type: string?
+    inputBinding:
+        prefix: -feats
   serial: 
     type: string?
     inputBinding:

progs/produce_enhanced_gff.cwl

@@ -0,0 +1,36 @@
+#!/usr/bin/env cwl-runner
+label: "Produce Roary input"
+doc: "Produce Prokka and Roary ready GFF enhancement = GFF + nuc FASTA"
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: cat
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entryname: separator.txt
+        entry: ${ return inputs.separator + '\n'; }
+inputs:
+  gff:
+    type: File?
+    inputBinding: 
+      position: 1
+  separator:
+    type: string?
+    default: '### FASTA'
+  separator_file:
+    type: string?
+    default: 'separator.txt'
+    inputBinding: 
+      position: 2
+  fasta:
+    type: File?
+    inputBinding:
+      position: 3
+  output_name:
+    type: string?
+    default: 'annot_with_genomic_fasta.gff'
+stdout: $(inputs.output_name)
+outputs:
+  output: 
+    type: stdout

progs/sparclbl.cwl

@@ -32,7 +32,7 @@ inputs:
         prefix: -d
   blastdb:
     type: string
-    default: cdd_split8
+    default: cdd
     inputBinding:
         prefix: -b
         valueFrom: $(inputs.b.path)/$(inputs.blastdb)

progs/unit_tests/test_produce_enhanced_gff/input.yaml

@@ -0,0 +1,6 @@
+gff:
+  class: File
+  location: annot.gff
+fasta:
+  class: File
+  location: annot.fna

scripts/pgap.py

@@ -403,7 +403,7 @@ def check_runtime_setting(settings, value, min):
             cmd = [self.params.docker_cmd, 'run', '-i', '-v', '{}:/cwd'.format(os.getcwd()), self.params.docker_image,
                    'bash', '-c', 'df -k /cwd /tmp ; ulimit -a ; cat /proc/{meminfo,cpuinfo}']
 
-        result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
+        result = subprocess.run(cmd, stdin=subprocess.DEVNULL, check=True, stdout=subprocess.PIPE)
         if result.returncode != 0:
             return
         output = result.stdout.decode('utf-8')
@@ -440,7 +440,21 @@ def check_runtime_setting(settings, value, min):
         #with open(filename, 'w', encoding='utf-8') as f:
             #f.write(u'{}\n'.format(settings))
         f.write(json.dumps(settings, sort_keys=True, indent=4))
+
+    def report_output_files(self, output, output_files):
+        # output_files = [
+        # {"file": "", "remove": True},
+        # ]
+        for output_pair in output_files:
+            fullname = os.path.join(output, output_pair["file"])
+            if os.path.exists(fullname) and os.path.getsize(fullname) > 0:
+                print(f'FILE: {output_pair["file"]}')
+                with open(fullname, 'r') as f:
+                    print(f.read())
+                if output_pair["remove"]:
+                    os.remove(fullname)
 
+
     def launch(self):
         cwllog = self.params.outputdir + '/cwltool.log'
         with open(cwllog, 'a', encoding="utf-8") as f:
@@ -475,6 +489,13 @@ def launch(self):
                 else:
                     print(f'{self.pipename} failed, docker exited with rc =', proc.returncode)
                     find_failed_step(cwllog)
+                output_files = [
+                        {"file": "final_asndisc_diag.xml", "remove": True},
+                        {"file": "final_asnval_diag.xml", "remove": True},
+                        {"file": "initial_asndisc_diag.xml", "remove": True},
+                        {"file": "initial_asnval_diag.xml", "remove": True}
+                    ]
+                self.report_output_files(self.params.args.output, output_files)
         return proc.returncode
 
 class Setup:
@@ -517,6 +538,10 @@ def __init__(self, args):
         self.test_genomes_path = '{}/test_genomes-{}'.format(self.install_dir, self.use_version)
         self.outputdir = self.get_output_dir()
         self.get_docker_info()
+        if self.docker_type == 'podman':
+            # see PGAPX-1073
+            self.docker_image = "docker.io/ncbi/{}:{}".format(self.repo, self.use_version)
+
         self.update_self()
         if (self.local_version != self.use_version) or not self.check_install_data():
             self.update()
@@ -628,7 +653,7 @@ def get_docker_info(self):
         if self.docker_cmd == None:
             sys.exit("Docker not found.")
 
-        version = subprocess.run([self.docker_cmd, '--version'], check=True, stdout=subprocess.PIPE).stdout.decode('utf-8')
+        version = subprocess.run([self.docker_cmd, '--version'], check=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL).stdout.decode('utf-8')
         self.docker_type = version.split(maxsplit=1)[0].lower()
         if self.docker_type not in docker_type_alternatives:
             self.docker_type = os.path.basename(os.path.realpath(self.docker_cmd)).lower()
@@ -667,7 +692,7 @@ def str2sec(s):
 
     def update(self):
         print(f"installation directory: {self.install_dir}")
-        subprocess.run(["/bin/df", "-k", self.install_dir])
+        subprocess.run(["/bin/df", "-k", self.install_dir], stdin=subprocess.DEVNULL)
         self.update_self()
         threads = list()
         docker_thread = mp.Process(target = self.install_docker, name='docker image pull')
@@ -693,7 +718,7 @@ def install_docker(self):
             sif = self.docker_image.replace("ncbi/pgap:", "pgap_") + ".sif"
             try:
                 subprocess.run([self.docker_cmd, 'sif', 'list', sif],
-                               check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                               check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
                 print("Singularity sif files exists, not updating.")
                 return
             except subprocess.CalledProcessError:
@@ -704,7 +729,7 @@ def install_docker(self):
         print('Downloading (as needed) Docker image {}'.format(docker_url))
         r=None;
         try:
-            r = subprocess.run([self.docker_cmd, 'pull', docker_url], check=True)
+            r = subprocess.run([self.docker_cmd, 'pull', docker_url], check=True, stdin=subprocess.DEVNULL)
             #print(r)
         except subprocess.CalledProcessError:
             print(r)
@@ -811,7 +836,12 @@ def write_version(self):
             f.write(u'{}\n'.format(self.use_version))
 
 
-
+def remove_empty_files(rootdir):
+    for f in os.listdir(rootdir):
+        fullname = os.path.join(rootdir, f)
+        if os.path.isfile(fullname) and os.path.getsize(fullname) == 0:
+            quiet_remove(fullname)
+
 def main():
     parser = argparse.ArgumentParser(description='Run PGAP.')
     parser.add_argument('input', nargs='?',
@@ -928,6 +958,7 @@ def main():
                         sys.exit(1)
                     else:
                         print("Ignoring")
+                    remove_empty_files(args.output)
 
             if not args.ani_only:
                 p = Pipeline(params, args.input, "pgap")
@@ -940,6 +971,7 @@ def main():
                         submol_modified = os.path.join(args.output, p.submol)
                         if os.path.exists(submol_modified):
                             os.remove(submol_modified)
+                remove_empty_files(args.output)
 
     except (Exception, KeyboardInterrupt) as exc:
         if args.debug:

wf_common.cwl

@@ -748,6 +748,12 @@ steps:
         nuc_fasta_name:
             default: annot.fna
     out: [nuc_fasta]
+  Generate_Annotation_Reports_gff_enhanced:
+    run: progs/produce_enhanced_gff.cwl
+    in:
+        gff: Generate_Annotation_Reports_gff/output
+        fasta: Generate_Annotation_Reports_nuc_fasta/nuc_fasta
+    out: [output]
   Generate_Annotation_Reports_prot_fasta:
     run: progs/asn2fasta.cwl
     in:
@@ -757,6 +763,28 @@ steps:
             prot_fasta_name:
                 default: annot.faa
     out: [prot_fasta]
+  Generate_Annotation_Reports_cds_nuc_fasta:
+    run: progs/asn2fasta.cwl
+    in:
+      i: Final_Bacterial_Package_sqn2gbent/output
+      type:
+        default: seq-entry
+      feats:
+        default: fasta_cds_na
+      fasta_name:
+        default: annot_cds_from_genomic.fna
+    out: [fasta]
+  Generate_Annotation_Reports_cds_prot_fasta:
+    run: progs/asn2fasta.cwl
+    in:
+      i: Final_Bacterial_Package_sqn2gbent/output
+      type:
+          default: seq-entry
+      feats:
+          default: fasta_cds_aa
+      fasta_name:
+          default: annot_translated_cds.faa
+    out: [fasta]
   Final_Bacterial_Package_std_validation:
     run: progs/std_validation.cwl
     in:
@@ -998,6 +1026,9 @@ outputs:
   gff:
     type: File
     outputSource:  Generate_Annotation_Reports_gff/output
+  gff_enhanced:
+    type: File
+    outputSource:  Generate_Annotation_Reports_gff_enhanced/output
   gbk:
     type: File
     outputSource:  Generate_Annotation_Reports_gbk/output
@@ -1007,6 +1038,12 @@ outputs:
   protein_fasta:
     type: File?
     outputSource: Generate_Annotation_Reports_prot_fasta/prot_fasta
+  cds_nucleotide_fasta:
+    type: File?
+    outputSource: Generate_Annotation_Reports_cds_nuc_fasta/fasta
+  cds_protein_fasta:
+    type: File?
+    outputSource: Generate_Annotation_Reports_cds_prot_fasta/fasta
   sqn:
     type: File
     outputSource:  add_checksum_sqn/output