diff --git a/flight_analysis.py b/flight_analysis.py index cfcf6c6..81b4ec0 100644 --- a/flight_analysis.py +++ b/flight_analysis.py @@ -2,6 +2,7 @@ import utils import os +import sys # logging logger_name = os.path.basename(__file__) @@ -59,7 +60,7 @@ def compute_iteration_time(start: datetime, end: datetime): def get_routes_df(routes: list): """ Returns a pandas dataframe with all the scraped results. - + Input: routes (list of lists) """ # compute number of total scrapes @@ -106,6 +107,8 @@ def get_routes_df(routes: list): if __name__ == "__main__": + SKIP_SAVE_TO_DB = len(sys.argv) > 1 and sys.argv[1] == "nodb" + # retreive routes to scrape routes = read_routes_from_file() @@ -125,7 +128,8 @@ def get_routes_df(routes: list): db.prepare_db_and_tables(overwrite_table=False) # add results to database - db.add_pandas_df_to_db(scraped_flights) - + if not SKIP_SAVE_TO_DB: + db.add_pandas_df_to_db(scraped_flights) + # handle backup here db.dump_database_to_sql_file() diff --git a/src/flight_analysis/database.py b/src/flight_analysis/database.py index 52eb0c5..c38c8af 100644 --- a/src/flight_analysis/database.py +++ b/src/flight_analysis/database.py @@ -183,10 +183,14 @@ def dump_database_to_sql_file(self): Returns: the path to the dumped .tar file. """ BACKUPS_FOLDER_NAME = "db_backups" - - # specify the backup file and folder name - date_str = datetime.now().strftime("%Y%m%d%H%M%S") + + # create the backup folder if it doesn't exist backup_folder = os.path.join(os.path.dirname(__file__), BACKUPS_FOLDER_NAME) + if not os.path.exists(backup_folder): + os.makedirs(backup_folder) + + # specify backup filename + date_str = datetime.now().strftime("%Y%m%d%H%M") backup_file = os.path.join(backup_folder, f"{date_str}_{self.db_name}.tar") # run the pg_dump command to create a backup