Version 0.4.0-alpha:

- Added the possibility to download the data tables for each plot.
- Improved the network and changed the save plot as image option, so that the plot will be saved using Matplotlib in higher resolution and in several formats.
- Run the calculations required for coverage plot in a different thread in the background, in order to present the initial plots display faster.

Author: inbalpaz

SynTrackerVis_app/config.py

@@ -1,9 +1,15 @@
 import bokeh.palettes as bp
+import colorcet as cc
 
-sampling_sizes = ['All_regions', '40', '60', '80', '100', '125', '150', '175', '200', '250', '300', '350', '400']
+col_set = ['Ref_genome', 'Sample1', 'Sample2', 'Region', 'Synteny_score']
+sampling_sizes = ['All', '40', '60', '80', '100', '125', '150', '175', '200', '250', '300', '350', '400']
 sampling_sizes_wo_all = ['40', '60', '80', '100', '125', '150', '175', '200', '250', '300', '350', '400']
+genomes_sorting_options = ['Number of compared pairs', 'Genome name']
+contig_sorting_options = ['Contig length', 'Contig name']
+catplot_types = ['Scatter (jitter) plot', 'Boxplot']
 min_pairs_for_all_regions = 100
 max_clustermap_cols = 120
+max_network_nodes = 300
 network_iterations_options = ['50', '100', '150', '200', '250', '300', '350', '400', '450', '500']
 network_thresholds_options = ['Mean APSS', 'Mean APSS+1 STD', 'Mean APSS+2 STD', 'Define another threshold']
 APSS_connections_threshold_default = 0.9
@@ -14,15 +20,19 @@
 
 ## CSS Styles ##
 normal_bar_color = "#B048B5"
-highlight_bar_color = "#ba2649"
+#highlight_bar_color = "#ba2649"
+highlight_bar_color = "#43BFC7"
 title_red_color = "#800517"
 title_purple_color = "#800080"
 title_blue_color = "#002060"
+same_color = "#F22C5D"
+diff_color = "#47A3E1"
+nodes_default_color = 'gray'
+
 
 main_area_style = {
     'width': "1200px",
     'padding': "20px",
-    #'background': "#b0e0e6",
 }
 
 single_multi_tabs_style = {
@@ -40,46 +50,48 @@
 }
 
 main_column_style = {
-    #'background': "#f9f9f9",
+    'background': "#f9f9f9",
     'padding': "20px",
-    #'padding': "0px 20px 10px 20px"
 }
 
 plot_card_style = {
     'background': "#ffffff",
     'width': "1150px",
 }
 
-## matplotlib patrameters
+secondary_button = {
+    'background': 'rgba(0, 128, 255, 0.5)',
+    'color': 'white'
+}
+
+# Export file formats
+matplotlib_file_formats = ['png', 'pdf', 'svg', 'eps']
+bokeh_file_formats = ['png', 'svg']
+
+# Colormaps
 clustermap_colormaps_list = ['Blues', 'Purples', 'Greens', 'Oranges', 'Reds', 'Greys',
                              'OrRd', 'PuRd', 'RdPu', 'BuPu', 'GnBu', 'PuBu', 'BuGn', 'YlGn',
                              'YlGnBu', 'PuBuGn', 'YlOrRd',
                              'spring', 'summer', 'autumn', 'winter', 'cool', 'Wistia']
-matplotlib_file_formats = ['png', 'pdf', 'svg', 'eps']
 
-## Bokeh patrameters
-Bokeh_categorical_colormap_dict = {
-    'Category20': bp.Category20[20],
-    'Category10': bp.Category10[10],
+categorical_colormap_dict = {
+    'cet_glasbey': cc.glasbey,
+    'cet_glasbey_light': cc.glasbey_light,
+    'cet_glasbey_category10': bp.Category10[10],
     'Set1': bp.Set1[9],
-    'Set3': bp.Set3[12],
-    'Spectral': bp.Spectral[11],
-    'Bokeh': bp.Bokeh[8],
+    'Set3': bp.Set3[12]
 }
-Bokeh_categorical_colormap_list = [bp.Category10[10], bp.Category20[20], bp.Pastel1[9], bp.Set1[9], bp.Set3[12],
-                                   bp.Spectral[11], bp.Bokeh[8], bp.Turbo256]
-Bokeh_continuous_colormap_dict = {
-    'Turbo256': bp.Turbo256,
-    'Plasma': bp.Plasma256,
-    'Viridis': bp.Viridis256,
+
+continuous_colormap_dict = {
+    'cet_rainbow4': cc.m_rainbow4,
+    'cet_isolum': cc.isolum,
+    'plasma': bp.Plasma256,
+    'viridis': bp.Viridis256,
     'Blues': bp.Blues256,
     'Reds': bp.Reds256,
     'Greens': bp.Greens256,
 }
-bokeh_file_formats = ['png', 'svg']
 
-normal_bar_color = "#B048B5"
-highlight_bar_color = "#43BFC7"
 
 
 

SynTrackerVis_app/data_manipulation_multi.py

@@ -38,8 +38,8 @@ def complete_metadata(score_per_region_df, metadata_df):
                 #new_row.append("NaN")
             metadata_df.loc[len(metadata_df)] = new_row
 
-    #print("\nMetadata after filling missing samples:")
-    #print(metadata_df)
+    print("\nMetadata after filling missing samples:")
+    print(metadata_df)
 
     # Create a dictionary to map the samples to feature values from metadata_df
     for feature in metadata_features_list:
@@ -63,6 +63,27 @@ def count_species_num(row, df):
     return species_num
 
 
+def create_sorted_by_pairs_genomes_list(score_per_region_all_genomes_df):
+    regions_num_per_pair_df = score_per_region_all_genomes_df[['Ref_genome', 'Sample1', 'Sample2', 'Synteny_score']]. \
+        groupby(['Ref_genome', 'Sample1', 'Sample2']).count().reset_index(). \
+        rename(columns={"Synteny_score": "Num_of_compared_regions"})
+
+    regions_num_per_pair_df['40'] = np.where(regions_num_per_pair_df['Num_of_compared_regions'] >= 40, 1, 0)
+
+    pairs_num_at_40_regions_df = regions_num_per_pair_df[['Ref_genome', '40']].groupby('Ref_genome').sum().\
+        sort_values('40', ascending=False).reset_index()
+    pairs_num_at_40_regions_df.columns.values[1] = "Number_of_pairs"
+
+    print("\ncreate_sorted_by_pairs_genomes_list:")
+    print(pairs_num_at_40_regions_df)
+
+    genomes_list_by_pairs_num = list(pairs_num_at_40_regions_df['Ref_genome'])
+    #print("\nGenomes list sorted by pairs number:")
+    #print(genomes_list_by_pairs_num)
+
+    return genomes_list_by_pairs_num
+
+
 def create_pairs_num_per_sampling_size(score_per_region_selected_genomes_df):
 
     regions_num_per_pair_df = score_per_region_selected_genomes_df[['Ref_genome', 'Sample1', 'Sample2', 'Synteny_score']].\
@@ -72,15 +93,15 @@ def create_pairs_num_per_sampling_size(score_per_region_selected_genomes_df):
 
     # Add a column for each subsampling size (to calculate how many pairs have results for at least this size)
     for size in config.sampling_sizes:
-        if size == 'All_regions':
+        if size == 'All':
             regions_num_per_pair_df[size] = np.where(regions_num_per_pair_df['Num_of_compared_regions'] >= 1,
                                                      1, 0)
         else:
             regions_num_per_pair_df[size] = np.where(regions_num_per_pair_df['Num_of_compared_regions'] >= int(size),
                                                      1, 0)
     #print(regions_num_per_pair_df)
 
-    pairs_num_per_sampling_size_df = regions_num_per_pair_df[['Ref_genome', 'All_regions', '40', '60', '80', '100',
+    pairs_num_per_sampling_size_df = regions_num_per_pair_df[['Ref_genome', 'All', '40', '60', '80', '100',
                                                               '125', '150', '175', '200', '250', '300', '350',
                                                               '400']].groupby('Ref_genome').sum().reset_index()
     #print(pairs_num_per_sampling_size_df)
@@ -90,7 +111,7 @@ def create_pairs_num_per_sampling_size(score_per_region_selected_genomes_df):
                                                         pairs_num_per_sampling_size_df[size], 0)
     #print(pairs_num_per_sampling_size_df)
 
-    summary_df = pairs_num_per_sampling_size_df[['All_regions', '40', '60', '80', '100', '125', '150', '175', '200',
+    summary_df = pairs_num_per_sampling_size_df[['All', '40', '60', '80', '100', '125', '150', '175', '200',
                                                  '250', '300', '350', '400']].sum().reset_index()
 
     summary_df.columns.values[0] = "Subsampled_regions"
@@ -109,7 +130,7 @@ def create_pairs_num_per_sampling_size(score_per_region_selected_genomes_df):
 def calculate_APSS_all_genomes_sampling_size(score_per_region_df, size):
 
     # Taking all available regions - no subsampling
-    if size == 'All_regions':
+    if size == 'All':
         avg_scores_one_size_df = score_per_region_df.groupby(['Ref_genome', 'Sample1', 'Sample2'])['Synteny_score'].\
             mean().reset_index().rename(columns={"Synteny_score": "APSS"})
 
@@ -131,18 +152,18 @@ def calculate_APSS_all_genomes_sampling_size(score_per_region_df, size):
     if not avg_scores_one_size_df.empty:
         avg_scores_one_size_df['Compared_regions'] = size
 
-    print("\ncalculate_APSS_all_genomes_sampling_size:")
-    print(avg_scores_one_size_df)
+    #print("\ncalculate_APSS_all_genomes_sampling_size:")
+    #print(avg_scores_one_size_df)
 
     # Filter out species with less than 10 pairs
     samples_per_genome_df = avg_scores_one_size_df[['Ref_genome', 'APSS']].groupby('Ref_genome').count().reset_index().\
         rename(columns={"APSS": "count"})
-    print(samples_per_genome_df)
+    #print(samples_per_genome_df)
     merged_df = avg_scores_one_size_df.merge(samples_per_genome_df[['Ref_genome', 'count']], on='Ref_genome',
                                              how='left')
     avg_scores_one_size_filtered_df = merged_df[merged_df['count'] >= 10].drop(columns='count')
-    print("\ncalculate_APSS_all_genomes_sampling_size after genomes filtering:")
-    print(avg_scores_one_size_filtered_df)
+    #print("\ncalculate_APSS_all_genomes_sampling_size after genomes filtering:")
+    #print(avg_scores_one_size_filtered_df)
 
     return avg_scores_one_size_filtered_df
 
@@ -152,8 +173,8 @@ def return_genomes_subset_APSS_selected_size_table(all_genomes_selected_size_APS
         all_genomes_selected_size_APSS_df[all_genomes_selected_size_APSS_df['Ref_genome'].isin(genomes_list)].\
         reset_index()
 
-    print("\nreturn_genomes_subset_APSS_selected_size_table:")
-    print(genomes_subset_selected_size_APSS_df)
+    #print("\nreturn_genomes_subset_APSS_selected_size_table:")
+    #print(genomes_subset_selected_size_APSS_df)
     return genomes_subset_selected_size_APSS_df
 
 

SynTrackerVis_app/data_manipulation_single.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import numpy as np
 import re
+import time
 import SynTrackerVis_app.config as config
 
 
@@ -23,7 +24,7 @@ def return_selected_genome_avg_table(avg_big_df, selected_genome):
 def calculate_avg_scores_selected_genome_size(score_per_region_selected_genome_df, genome, size):
 
     # Taking all available regions - no subsampling
-    if size == 'All_regions':
+    if size == 'All':
         avg_scores_one_size_df = score_per_region_selected_genome_df.groupby(['Sample1', 'Sample2'])['Synteny_score'].\
             mean().reset_index().rename(columns={"Synteny_score": "APSS"})
 
@@ -52,7 +53,18 @@ def calculate_avg_scores_selected_genome_size(score_per_region_selected_genome_d
 
     return avg_scores_one_size_df
 
+
+def count_samples_num(row, df):
+    condition_df = df[df[row['Subsampled_regions']] > 0]
+
+    unique_samples = pd.concat([condition_df['Sample1'], condition_df['Sample2']]).unique()
+    samples_num = len(unique_samples)
+
+    return samples_num
+
+
 def create_pairs_num_per_sampling_size(score_per_region_df):
+    print("\ncreate_pairs_num_per_sampling_size:")
 
     regions_num_per_pair_df = score_per_region_df[['Sample1', 'Sample2', 'Synteny_score']].\
         groupby(['Sample1', 'Sample2']).count().reset_index(). \
@@ -62,7 +74,7 @@ def create_pairs_num_per_sampling_size(score_per_region_df):
 
     # Add a column for each subsampling size (to calculate how many pairs have results for at least this size)
     for size in config.sampling_sizes:
-        if size == 'All_regions':
+        if size == 'All':
             regions_num_per_pair_df[size] = np.where(regions_num_per_pair_df['Num_of_compared_regions'] >= 1,
                                                      1, 0)
         else:
@@ -71,7 +83,7 @@ def create_pairs_num_per_sampling_size(score_per_region_df):
 
     #print(regions_num_per_pair_df)
 
-    pairs_num_per_sampling_size_df = regions_num_per_pair_df[['All_regions', '40', '60', '80', '100',
+    pairs_num_per_sampling_size_df = regions_num_per_pair_df[['All', '40', '60', '80', '100',
                                                               '125', '150', '175', '200', '250', '300', '350',
                                                               '400']].sum().reset_index()
     pairs_num_per_sampling_size_df.columns.values[0] = "Subsampled_regions"
@@ -82,51 +94,75 @@ def create_pairs_num_per_sampling_size(score_per_region_df):
     pairs_num_per_sampling_size_df['Pairs_lost_percent'] = \
         pairs_num_per_sampling_size_df['Pairs_lost_percent'].apply(lambda x: round(x, 2))
 
-    #print(pairs_num_per_sampling_size_df)
-    return pairs_num_per_sampling_size_df
-
-
-def create_score_per_region_sorted_contigs_table(score_per_region_df):
-    contigs_dict = dict()
-    contig_length_dict = dict()
+    # Calculate the number of samples in each sampling size
+    pairs_num_per_sampling_size_df['Number_of_samples'] = \
+        pairs_num_per_sampling_size_df.apply(lambda row: count_samples_num(row, regions_num_per_pair_df), axis=1)
 
-    # Split the 'Region' column into
-    score_per_region_df[['Contig_name', 'Position']] = score_per_region_df['Region'].str.extract(r'(\S+)_(\d+)_\d+')
-    score_per_region_df['Position'] = score_per_region_df['Position'].astype(int)
+    print(pairs_num_per_sampling_size_df)
 
-    # If the contig names contain numbers, sort them numerically
-    if re.search(r"^\S+_\d+$", score_per_region_df.iloc[0]['Contig_name']):
+    return pairs_num_per_sampling_size_df
 
-        # Create a temporary column 'contigs_sort' to sort the contig names numericlly
-        score_per_region_df['Contig_number'] = score_per_region_df['Contig_name'].str.extract(r'\S+_(\d+)')\
-            .astype(int)
 
-        contigs_list_by_name = list(score_per_region_df.sort_values('Contig_number').groupby(['Contig_name'],
-                                                                                             sort=False).groups)
+def return_sorted_contigs_lists(score_per_region_df):
+    '''
+    contigs_dict = {}
 
-    else:
-        contigs_list_by_name = list(score_per_region_df.groupby(['Contig_name']).groups)
+    region_list = list(score_per_region_df['Region'])
+    for region in region_list:
+        regex = r'(\S+)_(\d+)_\d+'
+        m = re.search(regex, region)
+        if m:
+            contig_name = m.group(1)
+            pos = m.group(2)
+            contigs_dict[contig_name] = int(pos)
 
-    #print("\ncreate_score_per_region_sorted_contigs_table:")
-    #print(score_per_region_df)
-    #print("\nContigs list sorted by name:")
-    #print(contigs_list_by_name)
+    # Sort the contigs dict by name
+    contigs_list_by_name = sorted(contigs_dict)
 
-    # Create a dictionary for the contigs, sorted by their names.
-    for contig in contigs_list_by_name:
+    # Sort the contigs dict by length
+    contigs_list_by_length = sorted(contigs_dict, key=lambda k: contigs_dict[k], reverse=True)
+    '''
 
-        score_per_region_contig_df = score_per_region_df[score_per_region_df['Contig_name'] == contig]
-        contigs_dict[contig] = score_per_region_contig_df[['Contig_name', 'Position', 'Synteny_score']]
+    before = time.time()
+    # Split the 'Region' column into Contig_name and Position
+    pattern = re.compile(r'(\S+)_(\d+)_\d+')
+    score_per_region_df[['Contig_name', 'Position']] = score_per_region_df['Region'].str.extract(pattern)
+    #score_per_region_df[['Contig_name', 'Position']] = score_per_region_df['Region'].str.extract(r'(\S+)_(\d+)_\d+')
+    score_per_region_df['Position'] = score_per_region_df['Position'].astype(int)
 
-        # Find contig length by the last position
-        score_per_region_contig_df = score_per_region_contig_df.sort_values('Position')
-        contig_length = score_per_region_contig_df.iloc[-1]['Position'] + config.region_length
-        contig_length_dict[contig] = contig_length
+    after = time.time()
+    duration = after - before
+    print("Extract position from region took " + str(duration) + " seconds.\n")
 
-    # Sort the contig lengths dict by the lengths and return a sorted list of names
-    #sorted_dict = dict(sorted(contig_length_dict.items(), key=lambda item: item[1], reverse=True))
-    sorted_by_length_list = sorted(contig_length_dict, key=contig_length_dict.get, reverse=True)
+    # Get a list of contigs, sorted by name
+    # If the contig names contain numbers, sort them numerically
+    #if re.search(r"^\S+_\d+$", score_per_region_df.iloc[0]['Contig_name']):
 
-    return contigs_dict, contigs_list_by_name, sorted_by_length_list
+        # Create a temporary column 'contigs_sort' to sort the contig names numericlly
+    #    score_per_region_df['Contig_number'] = score_per_region_df['Contig_name'].str.extract(r'\S+_(\d+)')\
+    #        .astype(int)
+
+    #    contigs_list_by_name = list(score_per_region_df.sort_values('Contig_number').groupby(['Contig_name'],
+    #                                                                                         sort=False).groups)
+
+    #else:
+    #    contigs_list_by_name = list(score_per_region_df.groupby(['Contig_name']).groups)
+
+    before = time.time()
+    contigs_list_by_name = list(score_per_region_df.groupby(['Contig_name']).groups)
+    after = time.time()
+    duration = after - before
+    print("Sort by name took " + str(duration) + " seconds.\n")
+
+    # Get a list of contigs, sorted by length
+    before = time.time()
+    contigs_list_by_length = list(score_per_region_df.sort_values('Position', ascending=False).groupby(['Contig_name'],
+                                                                                                       sort=False).
+                                  groups)
+    after = time.time()
+    duration = after - before
+    print("Sort by length took " + str(duration) + " seconds.\n")
+  
+    return contigs_list_by_name, contigs_list_by_length
 
 

SynTrackerVis_app/plots_multi_genomes.py

@@ -51,7 +51,7 @@ def plot_species_vs_sampling_size_bar(df, sampling_size, is_all_regions):
 
 def create_box_plot(avg_df, pvalues_df, color, use_metadata, feature, same_color, different_color):
 
-    print("\ncreate_box_plot: Feature is " + feature)
+    #print("\ncreate_box_plot: Feature is " + feature)
     #print("\nAPSS dataframe:")
     #print(avg_df)