Make reblocking default in single-sample workflows (#141)

Author: ldgauthier

pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.changelog.md

@@ -1,3 +1,8 @@
+# 1.5.2
+2021-11-01
+
+Updated GenotypeGVCFs to support reblocked GVCFs as inputs
+
 # 1.5.1
 2020-12-16
 

pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.wdl

@@ -6,7 +6,7 @@ import "../../../../../tasks/broad/JointGenotypingTasks.wdl" as Tasks
 # Joint Genotyping for hg38 Whole Genomes and Exomes (has not been tested on hg19)
 workflow JointGenotyping {
 
-  String pipeline_version = "1.5.1"
+  String pipeline_version = "1.5.2"
 
   input {
     File unpadded_intervals_file

pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotypingOnReblockedValidate.md

@@ -0,0 +1,24 @@
+It is expected that some variants will have annotation changes:
+- New version has higher ANs in some cases -- low quality genotypes retain more data
+- When ANs vary, DP can obviously vary, 
+- ACs may not agree for * alleles because of corrections in reblocking
+- InbreedingCoeff (and AS_InbreedingCoeff) can vary since it's likelihood-based and not count based like ExcessHet; this is especially noticeable at low GQs because likelihood is split more evenly across genotypes
+- RankSums can vary because of a histogram/median bug fix in PR #7131 implemented in versions 4.2.1.0 and subsequent; comparison to results of ldg_revGGVCFs with updated GATK make this easier
+- High QD values can vary because of the "correction" of remapping values > 35 back to a Gaussian centered at 30 (stdev 3)
+- DP may vary due to changes in reference depth associated with reference block merging
+- Strand bias annotations (mostly AS_SOR, but also SOR and occasionally FS) change when annotations are dropped for homozygous reference genotypes (annotations agree with updated strand bias counts)
+
+Reblocked callset will have a few more AC=2 hom-vars at low coverage sites because QUAL increases as hom-refs go to GQ0 and don't provide much evidence for reference
+
+Spanning deletion alleles may be missing in new output due to corrections in reblocking (--allow-missing-stars)
+In a few rare cases variants appear to be "dropped" in the reblocked output (this is the --allow-extra-alleles argument below)
+QUAL scores are highly sensitive to a variety of factors including CPU platform, but here they may vary more significantly because GQ0 genotypes with depth data are called as hom-refs
+
+The GATK branch ldg_VCFcomparator (4f0292abaeb7cabb527ca830048520e8aecdbde4) can take into account a lot of these
+ expected differences with a command like:
+ java -jar gatk.compare.jar VCFComparator -V:expected $truth -V:actual $test  --ignore-quals -R $hg38 --warn-on-errors \
+ --allow-extra-alleles --allow-missing-stars  --ignore-filters --ignore-attribute DP
+
+Lots of warnings about AN or InbreedingCoeff mismatches, but only exceptions that output VCs are of concern
+
+Exome tests needed VQSR indel Gaussians reduced to 3 (only 50 exomes -- not a lot of indels; --ignore-filters --ignore-attribute VQSLOD --ignore-attribute culprit)

pipelines/broad/dna_seq/germline/joint_genotyping/by_chromosome/JointGenotypingByChromosomePartOne.changelog.md

@@ -1,3 +1,8 @@
+# 1.4.2
+2021-11-01
+
+Task wdls used by JointGenotypingByChromosomePartOne were updated with changes that don't affect JointGenotypingByChromosomePartOne wdl
+
 # 1.4.1
 2020-12-16
 

pipelines/broad/dna_seq/germline/joint_genotyping/by_chromosome/JointGenotypingByChromosomePartOne.wdl

@@ -5,7 +5,7 @@ import "../../../../../../tasks/broad/JointGenotypingTasks.wdl" as Tasks
 # Joint Genotyping for hg38 Exomes and Whole Genomes (has not been tested on hg19)
 workflow JointGenotypingByChromosomePartOne {
 
-  String pipeline_version = "1.4.1"
+  String pipeline_version = "1.4.2"
 
   input {
     File unpadded_intervals_file

pipelines/broad/dna_seq/germline/joint_genotyping/by_chromosome/JointGenotypingByChromosomePartTwo.changelog.md

@@ -1,3 +1,8 @@
+# 1.4.2
+2021-11-01
+
+Task wdls used by JointGenotypingByChromosomePartTwo were updated with changes that don't affect JointGenotypingByChromosomePartTwo wdl
+
 # 1.4.1
 2020-12-16
 

pipelines/broad/dna_seq/germline/joint_genotyping/by_chromosome/JointGenotypingByChromosomePartTwo.wdl

@@ -5,7 +5,7 @@ import "../../../../../../tasks/broad/JointGenotypingTasks.wdl" as Tasks
 # Joint Genotyping for hg38 Exomes and Whole Genomes (has not been tested on hg19)
 workflow JointGenotypingByChromosomePartTwo {
 
-  String pipeline_version = "1.4.1"
+  String pipeline_version = "1.4.2"
 
   input {
     String callset_name

pipelines/broad/dna_seq/germline/joint_genotyping/exome/JointGenotyping.inputs.json

@@ -28,7 +28,7 @@
   "JointGenotyping.haplotype_database": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.haplotype_database.txt",
 
   "JointGenotyping.callset_name": "exomes_data_validation",
-  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/exome/joint_genotyping/gvcfs/round_three_sample_map",
+  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/exome/joint_genotyping/gvcfs/reblocked_sample_map",
 
   "JointGenotyping.small_disk" : 100,
   "JointGenotyping.medium_disk" : 200,

pipelines/broad/dna_seq/germline/joint_genotyping/exome/test_inputs/Plumbing/gather_vcfs_high_memory.json

@@ -28,13 +28,15 @@
   "JointGenotyping.haplotype_database": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.haplotype_database.txt",
 
   "JointGenotyping.callset_name": "small_callset_low_threshold",
-  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/joint_genotyping/exome/plumbing/callset/plumbing_sample_map",
+  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/joint_genotyping/exome/plumbing/callset/reblocked_plumbing_sample_map",
 
   "JointGenotyping.small_disk" : 100,
   "JointGenotyping.medium_disk" : 200,
   "JointGenotyping.large_disk" : 500,
   "JointGenotyping.huge_disk" : 2000,
 
   "JointGenotyping.gather_vcfs": true,
-  "JointGenotyping.snps_variant_recalibration_threshold": 50
+  "JointGenotyping.snps_variant_recalibration_threshold": 50,
+
+  "JointGenotyping.IndelsVariantRecalibrator.max_gaussians": 3
 }

pipelines/broad/dna_seq/germline/joint_genotyping/exome/test_inputs/Plumbing/gather_vcfs_low_memory.json

@@ -28,13 +28,15 @@
   "JointGenotyping.haplotype_database": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.haplotype_database.txt",
 
   "JointGenotyping.callset_name": "small_callset_high_threshold",
-  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/joint_genotyping/exome/plumbing/callset/plumbing_sample_map",
+  "JointGenotyping.sample_name_map" : "gs://broad-gotc-test-storage/joint_genotyping/exome/plumbing/callset/reblocked_plumbing_sample_map",
 
   "JointGenotyping.small_disk" : 100,
   "JointGenotyping.medium_disk" : 200,
   "JointGenotyping.large_disk" : 500,
   "JointGenotyping.huge_disk" : 2000,
 
   "JointGenotyping.gather_vcfs": true,
-  "JointGenotyping.snps_variant_recalibration_threshold": 500000
+  "JointGenotyping.snps_variant_recalibration_threshold": 500000,
+
+  "JointGenotyping.IndelsVariantRecalibrator.max_gaussians": 3
 }