v0.8.8

Author: frankvogt

files/lm_manh.png

@@ -17,7 +17,7 @@
 
 setup(
     name='vcf2gwas',
-    version='0.8.7',
+    version='0.8.8',
     description="Python API for comprehensive GWAS analysis using GEMMA",
     license="GNUv3",
     author="Frank Vogt",

files/lm_qq.png

@@ -23,13 +23,15 @@
 import sys
 import os
 import subprocess
+import time
 
-from vcf2gwas.parsing import *
+#from vcf2gwas.parsing import *
+from parsing import *
 from vcf2gwas.install import main as installer
 
 argvals = None
 
-def main(argvals=argvals):
+def main(timestamp, argvals=argvals):
 
     version = set_version_number()
     print(f"\nvcf2gwas v{version} \n")
@@ -39,13 +41,20 @@ def main(argvals=argvals):
     args.insert(0, 'python3.9')
     args.insert(1, os.path.join(os.path.dirname(__file__), 'starter.py'))
 
+    try:
+        args = delete_string(args, ['--timestamp'])
+    except:
+        pass
+    args.insert(2, timestamp)
+    args.insert(2, "--timestamp")
+    
     geno = P.set_geno()
     if geno == "test":
         source = os.path.join(os.path.dirname(__file__), 'starter.py')
         installer()
         vcf = os.path.join("input", "example.vcf.gz")
         pheno = os.path.join("input", "example.csv")
-        args = f'python3.9 {source} -v {vcf} -pf {pheno} -p 1 -lm'.split()
+        args = f'python3.9 {source} --timestamp {timestamp} -v {vcf} -pf {pheno} -p 1 -lm'.split()
 
     lm = P.set_lm()
     lmm = P.set_lmm()
@@ -59,19 +68,21 @@ def main(argvals=argvals):
     return process.returncode
 
 def run_main():
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
     try:
-        sys.exit(main())
+        sys.exit(main(timestamp))
     except KeyboardInterrupt as e:
         print("\nvcf2gwas interrupted")
         print("Cleaning up temporary files\n")
     finally:
-        shutil.rmtree("_vcf2gwas_temp", ignore_errors=True)
+        shutil.rmtree(f'_vcf2gwas_temp_{timestamp}', ignore_errors=True)
 
 if __name__ == '__main__':
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
     try:
-        sys.exit(main())
+        sys.exit(main(timestamp))
     except KeyboardInterrupt as e:
         print("\nvcf2gwas interrupted")
         print("Cleaning up temporary files\n")
     finally:
-        shutil.rmtree("_vcf2gwas_temp", ignore_errors=True)
+        shutil.rmtree(f'_vcf2gwas_temp_{timestamp}', ignore_errors=True)

files/lmm_manh.png

@@ -19,8 +19,10 @@
 along with vcf2gwas.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-from vcf2gwas.parsing import *
-from vcf2gwas.utils import *
+#from vcf2gwas.parsing import *
+#from vcf2gwas.utils import *
+from parsing import *
+from utils import *
 
 import sys
 import time
@@ -37,14 +39,13 @@
 
 ############################## Initialising Program ##############################
 
-timestamp = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
-timestamp2 = read_timestamp()
 start = time.perf_counter()
 # set argument parser
 P = Parser(argvals)
 out_dir = P.set_out_dir()
 out_dir2 = os.path.join(out_dir, "Output")
-dir_temp = "_vcf2gwas_temp"
+timestamp2 = P.set_timestamp()
+dir_temp = f'_vcf2gwas_temp_{timestamp2}'
 memory = P.set_memory()
 threads = P.set_threads()
 os.environ['NUMEXPR_MAX_THREADS'] = str(threads)
@@ -203,7 +204,7 @@
     msg = "No individuals left! Check if IDs in VCF and phenotype file are of the same format"
     raise_error(ValueError, msg, Log)
 
-file = open(os.path.join("_vcf2gwas_temp", "vcf2gwas_ind_count.txt"), 'a')
+file = open(os.path.join(dir_temp, "vcf2gwas_ind_count.txt"), 'a')
 file.write(f'{len(list1_org)-diff_num}\n')
 file.close()
 
@@ -335,7 +336,7 @@
 N_list = []
 path_list = []
 
-file = open(os.path.join("_vcf2gwas_temp", f"vcf2gwas_process_report_{pc_prefix}.txt"), 'a')
+file = open(os.path.join(dir_temp, f"vcf2gwas_process_report_{pc_prefix}.txt"), 'a')
 file.close()
 
 if model == None:
@@ -355,13 +356,15 @@
             i = "results"
 
         if X == [] and Y == []:
-            N = "1"
+            N = [1]
+            #N = "1"
         else:
-            N = str(x)
+            N = [x]
+            #N = str(x)
             x = x + 1
             if multi == True:
                 N = concat_lists(X, Y)
-                N = listtostring(N)
+                #N = listtostring(N)
 
         path_temp = None
         if model not in ("-gk", "-eigen"):
@@ -388,12 +391,12 @@
         executor.map(
             Gemma.run_gemma, prefix_list, prefix2_list, itertools.repeat(model), itertools.repeat(n), N_list, path_list, itertools.repeat(Log), itertools.repeat(filename), 
             itertools.repeat(filename2), itertools.repeat(pca), itertools.repeat(covar_file_name), i_list, itertools.repeat(i_list2),
-            itertools.repeat(burn), itertools.repeat(sampling), itertools.repeat(snpmax), itertools.repeat(pc_prefix)
+            itertools.repeat(burn), itertools.repeat(sampling), itertools.repeat(snpmax), itertools.repeat(pc_prefix), itertools.repeat(dir_temp)
         )
     timer_end = time.perf_counter()
     timer_total = round(timer_end - timer, 2)
-    Post_analysis.check_return_codes(pc_prefix)
-    Post_analysis.get_gemma_success(i_list, prefix2_list, path_list, columns, i_list2)
+    Post_analysis.check_return_codes(pc_prefix, dir_temp)
+    Post_analysis.get_gemma_success(i_list, prefix2_list, path_list, columns, i_list2, dir_temp)
     Log.print_log(f'\nGEMMA completed successfully (Duration: {runtime_format(timer_total)})\n')
 
     ############################## Processing and plotting ##############################
@@ -403,7 +406,7 @@
         itertools.repeat(top_ten), itertools.repeat(top_sig), itertools.repeat(top_all), itertools.repeat(Log), itertools.repeat(model), itertools.repeat(n), prefix2_list, path_list, 
         itertools.repeat(n_top), i_list, itertools.repeat(sigval), itertools.repeat(nolabel), itertools.repeat(noplot)
         ):
-        Post_analysis.run_postprocessing(top_ten, top_sig, top_all, Log, model, n, prefix2, path_temp, n_top, i, sigval, nolabel, noplot)
+        Post_analysis.run_postprocessing(top_ten, top_sig, top_all, Log, model, n, prefix2, path_temp, n_top, i, sigval, nolabel, noplot, dir_temp)
     Log.print_log("Analysis of GEMMA results completed successfully\n")
 
 ############################## Summary and Clean up ##############################

files/lmm_qq.png

@@ -31,7 +31,16 @@
     from psutil import virtual_memory
 
 def set_version_number():
-    return "0.8.7"
+    return "0.8.8"
+
+def delete_string(args, strings):
+
+    for string in strings:
+        temp_list = [i for i, e in enumerate(args) if e == string]
+        temp_list2 = [x+1 for x in temp_list]
+        temp_list = temp_list+temp_list2
+        args = [j for i, j in enumerate(args) if i not in temp_list]
+    return args
 
 def getArgs(argv=None):
     """Description:
@@ -44,6 +53,9 @@ def getArgs(argv=None):
     parser.add_argument(
         '--version', action='version', version='%(prog)s Version {}'.format(version)
     )
+    parser.add_argument(
+        '--timestamp', type=str
+    )
     parser.add_argument(
         "-v", "--vcf", metavar="<filename>", required=True, type=str, help="(required) Genotype .vcf or .vcf.gz filename"
     )
@@ -212,6 +224,9 @@ class Parser:
     def __init__(self, args):
         self.args = getArgs(args)
 
+    def set_timestamp(self):
+        return self.args.timestamp
+
     def set_geno(self):
         return self.args.vcf
 

setup.py

@@ -19,13 +19,16 @@
 along with vcf2gwas.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-from vcf2gwas.parsing import *
-from vcf2gwas.utils import *
+#from vcf2gwas.parsing import *
+#from vcf2gwas.utils import *
+from parsing import *
+from utils import *
 
 import time
 import os
 import sys
 import math
+import itertools
 import concurrent.futures
 import multiprocessing as mp
 from psutil import virtual_memory
@@ -43,17 +46,20 @@
 out_dir2 = os.path.join(out_dir, "Output")
 os.makedirs(out_dir2, exist_ok=True)
 
-timestamp = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
-timestamp2 = time.strftime("%Y%m%d_%H%M%S")
+timestamp = time.strftime("%a, %d %b %Y %H:%M", time.localtime())
+timestamp2 = P.set_timestamp()
+if timestamp2 == None:
+    # for testing
+    timestamp2 = time.strftime("%Y%m%d_%H%M%S")
+
 start = time.perf_counter()
 
 version = set_version_number()
 
-shutil.rmtree("_vcf2gwas_temp", ignore_errors=True)
-dir_temp = "_vcf2gwas_temp"
+shutil.rmtree(f'_vcf2gwas_temp_{timestamp2}', ignore_errors=True)
+dir_temp = f'_vcf2gwas_temp_{timestamp2}'
 qc_dir = os.path.join(dir_temp, f'QC_{timestamp2}')
 make_dir(qc_dir)
-write_timestamp(timestamp2)
 
 file = open(os.path.join(dir_temp, "vcf2gwas_process_report.txt"), 'a')
 file.close()
@@ -178,7 +184,7 @@
     path = os.path.join(out_dir2, model2)
 make_dir(path)
 
-pc_prefix = set_pc_prefix(pheno, covar, ".")
+pc_prefix = set_pc_prefix(pheno, covar, timestamp2)
 pc_prefix2 = set_pc_prefix(pheno, covar, "_")
 if model in ["-gk", "-eigen"]:
     pc_prefix2 = set_pc_prefix(pheno, covar, "")
@@ -642,12 +648,15 @@
 # for testing
 if args == []:
     args = argvals
+args = Starter.delete_string(args, ['--timestamp'])
 input_str = f'vcf2gwas {listtostring(args)}'
 args = Starter.delete_string(args, ['-v', '--vcf', '-T', '--threads', '-M', '--memory'])
 if covar == None:
     args = Starter.delete_string(args, ['-cf', '--cfile', '-c', '--covar'])
 args.insert(0, snp_file2)
 args.insert(0, "--vcf")
+args.insert(0, timestamp2)
+args.insert(0, "--timestamp")
 args.insert(0, 'python3.9')
 args.insert(1, os.path.join(os.path.dirname(__file__), 'analysis.py'))
 args.insert(2, '--memory')
@@ -702,8 +711,8 @@
 
 Log.print_log("\nStarting analysis..")
 with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context('fork'), max_workers=threads_org) as executor:
-        executor.map(Starter.run_vcf2gwas, args_list)
-Starter.check_return_codes(Log)
+        executor.map(Starter.run_vcf2gwas, args_list, itertools.repeat(dir_temp))
+Starter.check_return_codes(Log, dir_temp)
 
 #################### summary and clean-up ####################
 
@@ -721,7 +730,7 @@
         pc_prefix3 = set_pc_prefix(pheno, covar, "_")
         prefix_list.append(pc_prefix3)
     filenames, str_list = Summary.summarizer(path3, path2, pc_prefix3, snp_prefix, n_top, Log, prefix_list)
-    temp, file_dict = Summary.ind_summary(path2, filenames, str_list)
+    temp, file_dict = Summary.ind_summary(path2, filenames, str_list, dir_temp)
     filenames = temp[0]
     str_list = temp[1]
     filenames2 = []
@@ -863,13 +872,13 @@
     Y = pca_n
 
 # print summary and move log files
-snp_total, snp_sig = Starter.get_snpcounts()
+snp_total, snp_sig = Starter.get_snpcounts(dir_temp)
 if noplot == True:
     snp_sig = "-"
-sig_level = Starter.get_count("vcf2gwas_sig_level.txt")
-ind_count = Starter.get_count("vcf2gwas_ind_count.txt")
-gemma_count = Starter.get_count("vcf2gwas_ind_gemma.txt")
-failed_count, failed_list = Starter.get_gemma_fail()
+sig_level = Starter.get_count("vcf2gwas_sig_level.txt", dir_temp)
+ind_count = Starter.get_count("vcf2gwas_ind_count.txt", dir_temp)
+gemma_count = Starter.get_count("vcf2gwas_ind_gemma.txt", dir_temp)
+failed_count, failed_list = Starter.get_gemma_fail(dir_temp)
 Log.summary(
     snp_file, pheno_files, covar, X, Y, model2_temp, n, filename, min_af, A, B, 
     pca, keep, memory, threads, n_top, gene_file, species, gene_thresh, multi, umap_n, umapmetric, pca_n, 
@@ -880,4 +889,4 @@
 if model != None:
     shutil.move(os.path.join(path, f'vcf2gwas{pc_prefix}.log.txt'), os.path.join(path, f'vcf2gwas_{snp_prefix}{pc_prefix2}_{timestamp2}.log.txt'))
 
-shutil.rmtree("_vcf2gwas_temp", ignore_errors=True)
+shutil.rmtree(dir_temp, ignore_errors=True)