From bb212ae3cc1b87d94afac8f6fe90faabb0670d93 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 12 Sep 2025 17:03:55 +0100 Subject: [PATCH 001/100] Add separate config files for all connection methods. --- datashuttle/datashuttle_class.py | 5 +++ datashuttle/utils/aws.py | 3 +- datashuttle/utils/folders.py | 6 ++- datashuttle/utils/rclone.py | 43 +++++++++++++++---- tests/test_utils.py | 2 +- .../tests_transfers/aws/test_aws_transfer.py | 2 +- .../tests_transfers/aws/test_tui_setup_aws.py | 4 +- tests/tests_transfers/base_transfer.py | 2 +- .../gdrive/test_gdrive_transfer.py | 2 +- .../gdrive/test_tui_setup_gdrive.py | 2 +- 10 files changed, 53 insertions(+), 18 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 267990c82..7060c186f 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -844,6 +844,11 @@ def setup_ssh_connection(self) -> None: # Google Drive # ------------------------------------------------------------------------- + # TODO: this is going to be a massive pain because old config files will not work + # will need to re-set up all connections + # this can just be a breaking change, but will have to handle error nicely + # We could just move it from the config file, then show a warning + @check_configs_set def setup_gdrive_connection(self) -> None: """Set up a connection to Google Drive using the provided credentials. diff --git a/datashuttle/utils/aws.py b/datashuttle/utils/aws.py index 519a2b6db..baf53a5fc 100644 --- a/datashuttle/utils/aws.py +++ b/datashuttle/utils/aws.py @@ -11,7 +11,8 @@ def check_if_aws_bucket_exists(cfg: Configs) -> bool: The first part of`cfg["central_path"] should be an existing bucket name. """ output = rclone.call_rclone( - f"lsjson {cfg.get_rclone_config_name()}:", pipe_std=True + f"lsjson {cfg.get_rclone_config_name()}: {rclone.get_config_arg(cfg)}", + pipe_std=True, ) files_and_folders = json.loads(output.stdout) diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index bf3e6c973..a5ceca80d 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -695,10 +695,12 @@ def search_central_via_connection( If `True`, return the full filepath, otherwise return only the folder/file name. """ - rclone_config_name = cfg.get_rclone_config_name(cfg["connection_method"]) + rclone_config_name = cfg.get_rclone_config_name( + cfg["connection_method"] + ) # TODO: this is not good because we get the config name here and in get_config_arg output = rclone.call_rclone( - f'lsjson {rclone_config_name}:"{search_path.as_posix()}"', + f'lsjson {rclone_config_name}:"{search_path.as_posix()}" {rclone.get_config_arg(cfg)}', pipe_std=True, ) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index fe1909119..4cfaba053 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -11,6 +11,7 @@ import shlex import subprocess import tempfile +from pathlib import Path from subprocess import CompletedProcess from datashuttle.configs import canonical_configs @@ -208,15 +209,32 @@ def setup_rclone_config_for_ssh( f"host {cfg['central_host_id']} " f"user {cfg['central_host_username']} " f"port {canonical_configs.get_default_ssh_port()} " + f"{get_config_arg(cfg)} " f'-- key_pem "{key_escaped}"' ) - call_rclone(command, pipe_std=True) if log: log_rclone_config_output() +def get_config_path(): + """TODO PLACEHOLDER.""" + return ( + Path().home() / "AppData" / "Roaming" / "rclone" + ).as_posix() # # "$HOME/.config/rclone/rclone.conf") + + +def get_config_arg(cfg): + """TODO PLACEHOLDER.""" + cfg.get_rclone_config_name() # pass this? handle better... + + if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: + return f'--config "{get_config_path()}/{cfg.get_rclone_config_name()}.conf"' + else: + return "" + + def setup_rclone_config_for_gdrive( cfg: Configs, rclone_config_name: str, @@ -274,7 +292,8 @@ def setup_rclone_config_for_gdrive( f"{client_secret_key_value}" f"scope drive " f"root_folder_id {cfg['gdrive_root_folder_id']} " - f"{extra_args}" + f"{extra_args} " + f"{get_config_arg(cfg)}" ) return process @@ -322,7 +341,8 @@ def setup_rclone_config_for_aws( f"access_key_id {cfg['aws_access_key_id']} " f"secret_access_key {aws_secret_access_key} " f"region {aws_region}" - f"{location_constraint_key_value}", + f"{location_constraint_key_value} " + f"{get_config_arg(cfg)}", pipe_std=True, ) @@ -351,8 +371,11 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: else: tempfile_path = (cfg["central_path"] / filename).as_posix() + config_name = cfg.get_rclone_config_name() + output = call_rclone( - f"touch {cfg.get_rclone_config_name()}:{tempfile_path}", pipe_std=True + f"touch {config_name}:{tempfile_path} {get_config_arg(cfg)}", + pipe_std=True, ) if output.returncode != 0: utils.log_and_raise_error( @@ -360,7 +383,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: ) output = call_rclone( - f"delete {cfg.get_rclone_config_name()}:{tempfile_path}", pipe_std=True + f"delete {cfg.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", + pipe_std=True, ) if output.returncode != 0: utils.log_and_raise_error( @@ -368,7 +392,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: ) -def log_rclone_config_output() -> None: +def log_rclone_config_output() -> None: # TODO: remove or update this """Log the output from creating Rclone config.""" output = call_rclone("config file", pipe_std=True) utils.log( @@ -461,14 +485,14 @@ def transfer_data( output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.get_rclone_config_name()}:' - f'{central_filepath}" {extra_arguments}', + f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)}', ) elif upload_or_download == "download": output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{cfg.get_rclone_config_name()}:' - f'{central_filepath}" "{local_filepath}" {extra_arguments}', + f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)}', ) return output @@ -573,7 +597,8 @@ def perform_rclone_check( f"{rclone_args('check')} " f'"{local_filepath}" ' f'"{cfg.get_rclone_config_name()}:{central_filepath}"' - f" --combined -", + f"{get_config_arg(cfg)} " + f"--combined -", pipe_std=True, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index bb38c5851..5cb166cf4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -449,7 +449,7 @@ def recursive_search_central(project: DataShuttle): # -R flag searches recursively output = rclone.call_rclone( - f"lsjson -R {project.cfg.get_rclone_config_name()}:{path_}", + f"lsjson -R {project.cfg.get_rclone_config_name()}:{path_} {rclone.get_config_arg(project.cfg)}", pipe_std=True, ) diff --git a/tests/tests_transfers/aws/test_aws_transfer.py b/tests/tests_transfers/aws/test_aws_transfer.py index 348468a1b..a03af4657 100644 --- a/tests/tests_transfers/aws/test_aws_transfer.py +++ b/tests/tests_transfers/aws/test_aws_transfer.py @@ -28,7 +28,7 @@ def aws_setup(self, pathtable_and_project): yield [pathtable, project] rclone.call_rclone( - f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent}" + f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" ) @pytest.mark.parametrize( diff --git a/tests/tests_transfers/aws/test_tui_setup_aws.py b/tests/tests_transfers/aws/test_tui_setup_aws.py index e16b9e2f8..fc795353b 100644 --- a/tests/tests_transfers/aws/test_tui_setup_aws.py +++ b/tests/tests_transfers/aws/test_tui_setup_aws.py @@ -31,7 +31,9 @@ def central_path_and_project(self, setup_project_paths): yield central_path, project_name - rclone.call_rclone(f"purge central_{project_name}_aws:{central_path}") + rclone.call_rclone( + f"purge central_{project_name}_aws:{central_path}" + ) # TODO: I think this will fail, needs config @pytest.mark.asyncio async def test_aws_connection_setup(self, central_path_and_project): diff --git a/tests/tests_transfers/base_transfer.py b/tests/tests_transfers/base_transfer.py index 0b83fe108..c01b71141 100644 --- a/tests/tests_transfers/base_transfer.py +++ b/tests/tests_transfers/base_transfer.py @@ -242,7 +242,7 @@ def run_and_check_transfers( # Clean up, removing the temp directories and # resetting the project paths. rclone.call_rclone( - f"purge {project.cfg.get_rclone_config_name()}:{tmp_central_path.as_posix()}" + f"purge {project.cfg.get_rclone_config_name()}:{tmp_central_path.as_posix()} {rclone.get_config_arg(project.cfg)}" ) shutil.rmtree(tmp_local_path) diff --git a/tests/tests_transfers/gdrive/test_gdrive_transfer.py b/tests/tests_transfers/gdrive/test_gdrive_transfer.py index ca75f19a6..fd9effa0d 100644 --- a/tests/tests_transfers/gdrive/test_gdrive_transfer.py +++ b/tests/tests_transfers/gdrive/test_gdrive_transfer.py @@ -30,7 +30,7 @@ def gdrive_setup(self, pathtable_and_project): yield [pathtable, project] rclone.call_rclone( - f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent}" + f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" ) @pytest.mark.parametrize( diff --git a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py index 7c2f89faa..e7830005b 100644 --- a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py +++ b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py @@ -32,7 +32,7 @@ def central_path_and_project(self, setup_project_paths): yield central_path, project_name rclone.call_rclone( - f"purge central_{project_name}_gdrive:{central_path}" + f"purge central_{project_name}_gdrive:{central_path}" # TODO: I think this will fail ) @pytest.mark.parametrize("central_path_none", [True, False]) From f02d6f14c58c4f8d066c32420ed8f71c5a00bac2 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 24 Sep 2025 18:53:25 +0100 Subject: [PATCH 002/100] Add password machinery for windows. --- datashuttle/configs/config_class.py | 6 ++ datashuttle/datashuttle_class.py | 57 ++++++++++++++++ datashuttle/utils/rclone.py | 38 +++++++++-- datashuttle/utils/rclone_password.py | 96 +++++++++++++++++++++++++++ datashuttle/utils/test_file.xml | Bin 0 -> 2027 bytes 5 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 datashuttle/utils/rclone_password.py create mode 100644 datashuttle/utils/test_file.xml diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 67a8299c7..99f563c90 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -64,6 +64,12 @@ def __init__( self.hostkeys_path: Path self.project_metadata_path: Path + self.backend_has_password = { # TODO: REMOVE + "ssh": False, + "gdrive": False, + "aws": False, + } + def setup_after_load(self) -> None: """Set up the config after loading it.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 7060c186f..3caf70118 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -45,6 +45,7 @@ gdrive, getters, rclone, + rclone_password, ssh, utils, validation, @@ -112,6 +113,62 @@ def _set_attributes_after_config_load(self) -> None: self._make_project_metadata_if_does_not_exist() + def set_config_password(self): + """""" + # TODO: CHECK CONNECTION METHOD + connection_method = self.cfg["connection_method"] + + if self.cfg.backend_has_password[connection_method]: + raise RuntimeError( + "This config file already has a password set. First, use `remove_config_password` to remove it." + ) + + rclone_config_path = rclone.get_full_config_filepath( + self.cfg + ) # change name to rclone config becuase this is getting confusing! + + if not rclone_config_path.exists(): + raise RuntimeError( + f"Rclone config file for: {connection_method} was not found. " + f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" + ) + + password_filepath = rclone_password.get_password_filepath(self.cfg) + + if password_filepath.exists(): + password_filepath.unlink() + + rclone_password.save_credentials_password( + password_filepath, + ) + + rclone_password.set_config_password( + password_filepath, rclone.get_full_config_filepath(self.cfg) + ) + + self.cfg.backend_has_password[connection_method] = ( + True # HANDLE THIS PROPERLY + ) + print(self.cfg.backend_has_password[connection_method]) + + def remove_config_password(self): + """""" + # TODO: CHECK CONNECTION METHOD + connection_method = self.cfg["connection_method"] + + if self.cfg.backend_has_password[self.cfg["connection_method"]]: + raise RuntimeError( + f"The config for the current connection method: {self.cfg['connection_method']} does not have a password." + ) + config_filepath = rclone_password.get_password_filepath(self.cfg) + rclone_password.remove_config_password( + config_filepath, rclone.get_full_config_filepath(self.cfg) + ) + + self.cfg.backend_has_password[connection_method] = ( + False # HANDLE THIS PROPERLY + ) + # ------------------------------------------------------------------------- # Public Folder Makers # ------------------------------------------------------------------------- diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 4cfaba053..a4296c1af 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -15,7 +15,7 @@ from subprocess import CompletedProcess from datashuttle.configs import canonical_configs -from datashuttle.utils import utils +from datashuttle.utils import rclone_password, utils def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: @@ -202,6 +202,12 @@ def setup_rclone_config_for_ssh( """ key_escaped = private_key_str.replace("\n", "\\n") + rclone_config_filepath = get_full_config_filepath( + cfg + ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + command = ( f"config create " f"{rclone_config_name} " @@ -222,7 +228,11 @@ def get_config_path(): """TODO PLACEHOLDER.""" return ( Path().home() / "AppData" / "Roaming" / "rclone" - ).as_posix() # # "$HOME/.config/rclone/rclone.conf") + ) # # "$HOME/.config/rclone/rclone.conf") + + +def get_full_config_filepath(cfg: Configs) -> Path: + return get_config_path() / f"{cfg.get_rclone_config_name()}.conf" def get_config_arg(cfg): @@ -230,11 +240,20 @@ def get_config_arg(cfg): cfg.get_rclone_config_name() # pass this? handle better... if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: - return f'--config "{get_config_path()}/{cfg.get_rclone_config_name()}.conf"' + return f'--config "{get_full_config_filepath(cfg)}"' else: return "" +def set_password(cfg, password: str): + subprocess.run( + f"rclone config encryption set {get_config_arg(cfg)}", text=True + ) + + +# def remove_password(): + + def setup_rclone_config_for_gdrive( cfg: Configs, rclone_config_name: str, @@ -481,20 +500,29 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) + if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter + print("SET") + config_filepath = rclone_password.get_password_filepath(cfg) + rclone_password.set_credentials_as_password_command(config_filepath) + if upload_or_download == "upload": output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.get_rclone_config_name()}:' - f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)}', + f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) elif upload_or_download == "download": output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{cfg.get_rclone_config_name()}:' - f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)}', + f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) + if cfg.backend_has_password[cfg["connection_method"]]: + print("REMOVED") + rclone_password.remove_credentials_as_password_command() + return output diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py new file mode 100644 index 000000000..e160adc14 --- /dev/null +++ b/datashuttle/utils/rclone_password.py @@ -0,0 +1,96 @@ +import os +import platform +import shutil +import subprocess +from pathlib import Path + +from datashuttle.configs import canonical_folders + + +def get_password_filepath( + cfg, +): # Configs # TOOD: datashuttle_path should be on configs? + """""" + assert cfg["connection_method"] in ["aws", "gdrive", "ssh"], ( + "password should only be set for ssh, aws, gdrive." + ) + + base_path = canonical_folders.get_datashuttle_path() / "credentials" + + base_path.mkdir(exist_ok=True, parents=True) + + return base_path / f"{cfg.get_rclone_config_name()}.xml" + + +def save_credentials_password(password_filepath: Path): + """""" + if platform.system() == "Windows": + # $env:APPDATA\\rclone\\rclone-credential.xml + shell = shutil.which("powershell") + if not shell: + raise RuntimeError( + "powershell.exe not found in PATH (need Windows PowerShell 5.1)." + ) + + ps_cmd = ( + "Add-Type -AssemblyName System.Web; " + "New-Object PSCredential 'rclone', " + "(ConvertTo-SecureString ([System.Web.Security.Membership]::GeneratePassword(40,10)) -AsPlainText -Force) " + f"| Export-Clixml -LiteralPath '{password_filepath}'" + ) + + # run it + subprocess.run([shell, "-NoProfile", "-Command", ps_cmd], check=True) + + +def set_credentials_as_password_command(password_filepath: Path): + """""" + # if platform.system() == "Windows": + # filepath = Path(filepath).resolve() + + shell = shutil.which("powershell") + if not shell: + raise RuntimeError("powershell.exe not found in PATH") + + # Escape single quotes inside PowerShell string by doubling them + # safe_path = str(filepath).replace("'", "''") + + cmd = ( + f'{shell} -NoProfile -Command "Write-Output (' + f"[System.Runtime.InteropServices.Marshal]::PtrToStringAuto(" + f"[System.Runtime.InteropServices.Marshal]::SecureStringToBSTR(" + f"(Import-Clixml -LiteralPath '{password_filepath.as_posix()}' ).Password)))\"" + ) + os.environ["RCLONE_PASSWORD_COMMAND"] = cmd + + +def set_config_password(password_filepath: Path, config_filepath: Path): + """""" + assert password_filepath.exists(), ( + "password file not found at point of config creation." + ) + + set_credentials_as_password_command(password_filepath) + + subprocess.run( + f"rclone config encryption set --config {config_filepath.as_posix()} " + ) + + remove_credentials_as_password_command() + + +# TODO: HANDLE ERRORS +def remove_config_password(password_filepath: Path, config_filepath: Path): + """""" + set_credentials_as_password_command(Path(password_filepath)) + subprocess.run( + rf"rclone config encryption remove --config {config_filepath.as_posix()}" + ) + + # TODO: HANDLE ERRORS + remove_credentials_as_password_command() + + +def remove_credentials_as_password_command(): + if "RCLONE_PASSWORD_COMMAND" in os.environ: + os.environ.pop("RCLONE_PASSWORD_COMMAND") diff --git a/datashuttle/utils/test_file.xml b/datashuttle/utils/test_file.xml new file mode 100644 index 0000000000000000000000000000000000000000..ef0d66c08453304822791f2221d0b411752dc772 GIT binary patch literal 2027 zcmcJQ$!-%t5Qgi_Q$%?H+vAz>f7z5g((8ud$1>_!5iQ!mVNx5Ai0>qlivgy|kaB^F8ikjdYjfcd>^a<8^eQ zUCiPRANMhZ@3619+2U=-TB6zEyy3o!%_g4y#M5RCb)z3WsDktNJTBuEbOs%8p2a7$ zm+?O42|@2jD{J1w1dD&^9ds|_I(~p~pOAOr1Lz-Ex9Plq7x&xv#xC|Ld#+F?uCRSa zh6~VS%@y)BXuEOq7kXp86ff6*L1p&O+GSsp6K@yz-w=II^c5Iyk^iqePU9SUk~6wW z9e$*OZMWKdugQCoI`tV1=W9ZR?&0k2maNz0@xa+UuV3)ddbz@S2R&i6rRsX=F{94S zf#z@{JdcsjnHA+l)bQ>_k`3{r{)|zcXSMBXpes@5JC{MNz}q9jmQR zvoe>GB`^A10TJufJjV%%+7(i3>hvy>#|o`7$*q@DX+4yQ)7dw#L#M~+8eh~Uu^K#D zH7QTLd8s8XyHzKdzI`iGPCb4{*e+9EBX%R&72`R5cC|>^+9502q&uff`$38^#9x&x z+vza_W2iBNNk$Wz>C6s45-;2~4%M)r(%RgVl X({G~R$>+Yg-sk_4{2gx3>C5~ZEMXNi literal 0 HcmV?d00001 From bcc5da430c8970ff0f498f2184fd8d944a905585 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 29 Sep 2025 17:15:50 +0100 Subject: [PATCH 003/100] Prototype input for set up ssh connection. --- datashuttle/datashuttle_class.py | 35 +++++++++++++++++++++++++++++++- datashuttle/utils/rclone.py | 4 +++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 3caf70118..7d2e490fc 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -125,7 +125,7 @@ def set_config_password(self): rclone_config_path = rclone.get_full_config_filepath( self.cfg - ) # change name to rclone config becuase this is getting confusing! + ) # change name to rclone config because this is getting confusing! if not rclone_config_path.exists(): raise RuntimeError( @@ -887,10 +887,19 @@ def setup_ssh_connection(self) -> None: self._setup_rclone_central_ssh_config(private_key_str, log=True) + config_filepath = rclone_password.get_password_filepath(self.cfg) + rclone_password.set_credentials_as_password_command( + config_filepath + ) + + print("Checking write permissions on the `central_path`...") + rclone.check_successful_connection_and_raise_error_on_fail( self.cfg ) + rclone_password.remove_credentials_as_password_command() + utils.log_and_message( "SSH key pair setup successfully. SSH key saved to the RClone config file." ) @@ -1632,6 +1641,15 @@ def _make_project_metadata_if_does_not_exist(self) -> None: def _setup_rclone_central_ssh_config( self, private_key_str: str, log: bool ) -> None: + input_ = input( + f"Your SSH key will be stored in the rclone config at:\n " + f"{rclone.get_full_config_filepath(self.cfg)}.\n\n" + f"Would you like to set a password using Windows credential manager? " + f"Press 'y' to set password or leave blank to skip." + ) + + set_password = input_ == "y" + rclone.setup_rclone_config_for_ssh( self.cfg, self.cfg.get_rclone_config_name("ssh"), @@ -1639,6 +1657,21 @@ def _setup_rclone_central_ssh_config( log=log, ) + if set_password: + try: + self.set_config_password() + except BaseException as e: + print(e) + # THIS PATH IS WRONG + config_path = rclone.get_full_config_filepath(self.cfg) + + raise RuntimeError( + f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" + f"Use set_config_password()` to attempt to set the password again (see stacktrace above). " + ) + + print("Password set successfully") + def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( self.cfg.get_rclone_config_name("local_filesystem"), diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a4296c1af..673f2ddaf 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -502,7 +502,9 @@ def transfer_data( if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter print("SET") - config_filepath = rclone_password.get_password_filepath(cfg) + config_filepath = rclone_password.get_password_filepath( + cfg + ) # TODO: ONE FUNCTION OR INCORPORATE INTO SINGLE FUNCTION rclone_password.set_credentials_as_password_command(config_filepath) if upload_or_download == "upload": From 0c4017e352ab998c7d19f2a5f165af9b8d20a5af Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 29 Sep 2025 18:16:18 +0100 Subject: [PATCH 004/100] Refactor rclone call functions to handle password. --- datashuttle/datashuttle_class.py | 11 ++--- datashuttle/utils/aws.py | 3 +- datashuttle/utils/folders.py | 3 +- datashuttle/utils/rclone.py | 79 ++++++++++++++++++++++++-------- 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 7d2e490fc..fc904e304 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -887,19 +887,12 @@ def setup_ssh_connection(self) -> None: self._setup_rclone_central_ssh_config(private_key_str, log=True) - config_filepath = rclone_password.get_password_filepath(self.cfg) - rclone_password.set_credentials_as_password_command( - config_filepath - ) - print("Checking write permissions on the `central_path`...") rclone.check_successful_connection_and_raise_error_on_fail( self.cfg ) - rclone_password.remove_credentials_as_password_command() - utils.log_and_message( "SSH key pair setup successfully. SSH key saved to the RClone config file." ) @@ -957,7 +950,9 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) - rclone.await_call_rclone_with_popen_raise_on_fail(process, log=True) + rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( + self.cfg, process, log=True + ) rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) diff --git a/datashuttle/utils/aws.py b/datashuttle/utils/aws.py index baf53a5fc..9bf1a27cc 100644 --- a/datashuttle/utils/aws.py +++ b/datashuttle/utils/aws.py @@ -10,7 +10,8 @@ def check_if_aws_bucket_exists(cfg: Configs) -> bool: The first part of`cfg["central_path"] should be an existing bucket name. """ - output = rclone.call_rclone( + output = rclone.call_rclone_for_central_connection( + cfg, f"lsjson {cfg.get_rclone_config_name()}: {rclone.get_config_arg(cfg)}", pipe_std=True, ) diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index a5ceca80d..2b31e64b9 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -699,7 +699,8 @@ def search_central_via_connection( cfg["connection_method"] ) # TODO: this is not good because we get the config name here and in get_config_arg - output = rclone.call_rclone( + output = rclone.call_rclone_for_central_connection( + cfg, f'lsjson {rclone_config_name}:"{search_path.as_posix()}" {rclone.get_config_arg(cfg)}', pipe_std=True, ) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 673f2ddaf..4b1f10596 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -48,7 +48,17 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: return output -def call_rclone_through_script(command: str) -> CompletedProcess: +def call_rclone_for_central_connection( + cfg, command: str, pipe_std: bool = False +) -> CompletedProcess: + return run_function_that_may_require_central_connection_password( + cfg, lambda: call_rclone(command, pipe_std) + ) + + +def call_rclone_through_script_for_central_connection( + cfg, command: str +) -> CompletedProcess: """Call rclone through a script. This is to avoid limits on command-line calls (in particular on Windows). @@ -84,13 +94,17 @@ def call_rclone_through_script(command: str) -> CompletedProcess: if system != "Windows": os.chmod(tmp_script_path, 0o700) - output = subprocess.run( + lambda_func = lambda: subprocess.run( [tmp_script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, ) + output = run_function_that_may_require_central_connection_password( + cfg, lambda_func + ) + if output.returncode != 0: prompt_rclone_download_if_does_not_exist() @@ -100,7 +114,9 @@ def call_rclone_through_script(command: str) -> CompletedProcess: return output -def call_rclone_with_popen(command: str) -> subprocess.Popen: +def call_rclone_with_popen_for_central_connection( + command: str, +) -> subprocess.Popen: """Call rclone using `subprocess.Popen` for control over process termination. It is not possible to kill a process while running it using `subprocess.run`. @@ -116,7 +132,7 @@ def call_rclone_with_popen(command: str) -> subprocess.Popen: return process -def await_call_rclone_with_popen_raise_on_fail( +def await_call_rclone_with_popen_for_central_connection_raise_on_fail( process: subprocess.Popen, log: bool = True ): """Await rclone the subprocess.Popen call. @@ -124,7 +140,11 @@ def await_call_rclone_with_popen_raise_on_fail( Calling `process.communicate()` waits for the process to complete and returns the stdout and stderr. """ - stdout, stderr = process.communicate() + lambda_func = lambda: process.communicate() + + stdout, stderr = run_function_that_may_require_central_connection_password( + cfg, lambda_func + ) if process.returncode != 0: utils.log_and_raise_error(stderr.decode("utf-8"), ConnectionError) @@ -133,6 +153,24 @@ def await_call_rclone_with_popen_raise_on_fail( log_rclone_config_output() +def run_function_that_may_require_central_connection_password( + cfg, lambda_func +): + """ """ + set_password = cfg.backend_has_password[cfg["connection_method"]] + + if set_password: + config_filepath = rclone_password.get_password_filepath(cfg) + rclone_password.set_credentials_as_password_command(config_filepath) + + results = lambda_func() + + if set_password: + rclone_password.remove_credentials_as_password_command() + + return results + + # ----------------------------------------------------------------------------- # Setup # ----------------------------------------------------------------------------- @@ -262,7 +300,7 @@ def setup_rclone_config_for_gdrive( ) -> subprocess.Popen: """Set up rclone config for connections to Google Drive. - This function uses `call_rclone_with_popen` instead of `call_rclone`. This + This function uses `call_rclone_with_popen_for_central_connection` instead of `call_rclone`. This is done to have more control over the setup process in case the user wishes to cancel the setup. Since the rclone setup for google drive uses a local web server for authentication to google drive, the running process must be killed before the @@ -303,7 +341,7 @@ def setup_rclone_config_for_gdrive( else "" ) - process = call_rclone_with_popen( + process = call_rclone_with_popen_for_central_connection( f"config create " f"{rclone_config_name} " f"drive " @@ -392,7 +430,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: config_name = cfg.get_rclone_config_name() - output = call_rclone( + output = call_rclone_for_central_connection( + cfg, f"touch {config_name}:{tempfile_path} {get_config_arg(cfg)}", pipe_std=True, ) @@ -401,7 +440,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output.stderr.decode("utf-8"), ConnectionError ) - output = call_rclone( + output = call_rclone_for_central_connection( + cfg, f"delete {cfg.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", pipe_std=True, ) @@ -500,22 +540,24 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) - if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter - print("SET") - config_filepath = rclone_password.get_password_filepath( - cfg - ) # TODO: ONE FUNCTION OR INCORPORATE INTO SINGLE FUNCTION - rclone_password.set_credentials_as_password_command(config_filepath) + # if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter + # print("SET") + # config_filepath = rclone_password.get_password_filepath( + # cfg + # ) # TODO: ONE FUNCTION OR INCORPORATE INTO SINGLE FUNCTION + # rclone_password.set_credentials_as_password_command(config_filepath) if upload_or_download == "upload": - output = call_rclone_through_script( + output = call_rclone_through_script_for_central_connection( + cfg, f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.get_rclone_config_name()}:' f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) elif upload_or_download == "download": - output = call_rclone_through_script( + output = call_rclone_through_script_for_central_connection( + cfg, f"{rclone_args('copy')} " f'"{cfg.get_rclone_config_name()}:' f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error @@ -623,7 +665,8 @@ def perform_rclone_check( "central", top_level_folder ).parent.as_posix() - output = call_rclone( + output = call_rclone_for_central_connection( + cfg, f"{rclone_args('check')} " f'"{local_filepath}" ' f'"{cfg.get_rclone_config_name()}:{central_filepath}"' From 9581d5a5976cba0561f7e2fd4ec04643edd2b87e Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 30 Sep 2025 17:10:28 +0100 Subject: [PATCH 005/100] Adding to google drive. --- datashuttle/configs/config_class.py | 27 ++++++++-- datashuttle/datashuttle_class.py | 79 ++++++++++++++++------------ datashuttle/utils/rclone.py | 8 +-- datashuttle/utils/rclone_password.py | 4 +- 4 files changed, 72 insertions(+), 46 deletions(-) diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 99f563c90..b227d4667 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -64,11 +64,24 @@ def __init__( self.hostkeys_path: Path self.project_metadata_path: Path - self.backend_has_password = { # TODO: REMOVE - "ssh": False, - "gdrive": False, - "aws": False, - } + self.rclone_password_state_file_path = ( + self.file_path.parent / "rclone_ps_state.yaml" + ) + self.rclone_has_password = {} + self.setup_rclone_has_password() + + def setup_rclone_has_password(self): + """""" + if self.rclone_password_state_file_path.is_file(): + with open(self.rclone_password_state_file_path, "r") as file: + self.rclone_has_password = yaml.full_load(file) + else: + self.rclone_has_password = { + "ssh": False, + "gdrive": False, + "aws": False, + } + self.save_rclone_password_state() def setup_after_load(self) -> None: """Set up the config after loading it.""" @@ -170,6 +183,10 @@ def update_config_for_backward_compatability_if_required( if config_dict["connection_method"] is None: config_dict["connection_method"] = "local_only" + def save_rclone_password_state(self): + with open(self.rclone_password_state_file_path, "w") as file: + yaml.dump(self.rclone_has_password, file) + # ------------------------------------------------------------------------- # Utils # ------------------------------------------------------------------------- diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index fc904e304..31048c3e3 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -113,14 +113,40 @@ def _set_attributes_after_config_load(self) -> None: self._make_project_metadata_if_does_not_exist() - def set_config_password(self): + def _try_set_rclone_password(self): # TODO: BETTER NAME! + """""" + input_ = input( + f"Your SSH key will be stored in the rclone config at:\n " + f"{rclone.get_full_config_filepath(self.cfg)}.\n\n" + f"Would you like to set a password using Windows credential manager? " + f"Press 'y' to set password or leave blank to skip." + ) + + set_password = input_ == "y" + + if set_password: + try: + self.set_rclone_password() + except BaseException as e: + print(e) + # THIS PATH IS WRONG + config_path = rclone.get_full_config_filepath(self.cfg) + + raise RuntimeError( + f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" + f"Use set_rclone_password()` to attempt to set the password again (see stacktrace above). " + ) + + print("Password set successfully") + + def set_rclone_password(self): """""" # TODO: CHECK CONNECTION METHOD connection_method = self.cfg["connection_method"] - if self.cfg.backend_has_password[connection_method]: + if self.cfg.rclone_has_password[connection_method]: raise RuntimeError( - "This config file already has a password set. First, use `remove_config_password` to remove it." + "This config file already has a password set. First, use `remove_rclone_password` to remove it." ) rclone_config_path = rclone.get_full_config_filepath( @@ -142,32 +168,36 @@ def set_config_password(self): password_filepath, ) - rclone_password.set_config_password( + rclone_password.set_rclone_password( password_filepath, rclone.get_full_config_filepath(self.cfg) ) - self.cfg.backend_has_password[connection_method] = ( + self.cfg.rclone_has_password[connection_method] = ( True # HANDLE THIS PROPERLY ) - print(self.cfg.backend_has_password[connection_method]) + self.cfg.save_rclone_password_state() + print(self.cfg.rclone_has_password[connection_method]) - def remove_config_password(self): + def remove_rclone_password(self): """""" # TODO: CHECK CONNECTION METHOD connection_method = self.cfg["connection_method"] - if self.cfg.backend_has_password[self.cfg["connection_method"]]: + if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: raise RuntimeError( - f"The config for the current connection method: {self.cfg['connection_method']} does not have a password." + f"The config for the current connection method: {self.cfg['connection_method']} does not have a password. Cannot remove." ) config_filepath = rclone_password.get_password_filepath(self.cfg) - rclone_password.remove_config_password( + rclone_password.remove_rclone_password( config_filepath, rclone.get_full_config_filepath(self.cfg) ) - self.cfg.backend_has_password[connection_method] = ( + self.cfg.rclone_has_password[connection_method] = ( False # HANDLE THIS PROPERLY ) + self.cfg.save_rclone_password_state() + + print("SAY SOMETHING LIKE PASSWORD REMOVED") # ------------------------------------------------------------------------- # Public Folder Makers @@ -950,6 +980,8 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) + self._try_set_rclone_password() + rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( self.cfg, process, log=True ) @@ -1636,36 +1668,13 @@ def _make_project_metadata_if_does_not_exist(self) -> None: def _setup_rclone_central_ssh_config( self, private_key_str: str, log: bool ) -> None: - input_ = input( - f"Your SSH key will be stored in the rclone config at:\n " - f"{rclone.get_full_config_filepath(self.cfg)}.\n\n" - f"Would you like to set a password using Windows credential manager? " - f"Press 'y' to set password or leave blank to skip." - ) - - set_password = input_ == "y" - rclone.setup_rclone_config_for_ssh( self.cfg, self.cfg.get_rclone_config_name("ssh"), private_key_str, log=log, ) - - if set_password: - try: - self.set_config_password() - except BaseException as e: - print(e) - # THIS PATH IS WRONG - config_path = rclone.get_full_config_filepath(self.cfg) - - raise RuntimeError( - f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" - f"Use set_config_password()` to attempt to set the password again (see stacktrace above). " - ) - - print("Password set successfully") + self._try_set_rclone_password() def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 4b1f10596..5b390b396 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -133,7 +133,7 @@ def call_rclone_with_popen_for_central_connection( def await_call_rclone_with_popen_for_central_connection_raise_on_fail( - process: subprocess.Popen, log: bool = True + cfg, process: subprocess.Popen, log: bool = True ): """Await rclone the subprocess.Popen call. @@ -157,7 +157,7 @@ def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ - set_password = cfg.backend_has_password[cfg["connection_method"]] + set_password = cfg.rclone_has_password[cfg["connection_method"]] if set_password: config_filepath = rclone_password.get_password_filepath(cfg) @@ -540,7 +540,7 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) - # if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter + # if cfg.rclone_has_password[cfg["connection_method"]]: # TODO: one getter # print("SET") # config_filepath = rclone_password.get_password_filepath( # cfg @@ -563,7 +563,7 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.backend_has_password[cfg["connection_method"]]: + if cfg.rclone_has_password[cfg["connection_method"]]: print("REMOVED") rclone_password.remove_credentials_as_password_command() diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index e160adc14..21924b2eb 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -64,7 +64,7 @@ def set_credentials_as_password_command(password_filepath: Path): os.environ["RCLONE_PASSWORD_COMMAND"] = cmd -def set_config_password(password_filepath: Path, config_filepath: Path): +def set_rclone_password(password_filepath: Path, config_filepath: Path): """""" assert password_filepath.exists(), ( "password file not found at point of config creation." @@ -80,7 +80,7 @@ def set_config_password(password_filepath: Path, config_filepath: Path): # TODO: HANDLE ERRORS -def remove_config_password(password_filepath: Path, config_filepath: Path): +def remove_rclone_password(password_filepath: Path, config_filepath: Path): """""" set_credentials_as_password_command(Path(password_filepath)) subprocess.run( From ba2759140fba64faa8d788dafbde35259ec335a7 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 30 Sep 2025 18:34:05 +0100 Subject: [PATCH 006/100] Added password to google dirve. --- datashuttle/datashuttle_class.py | 16 +++++++++++++--- datashuttle/utils/rclone.py | 10 ++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 31048c3e3..e5fe4b105 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -116,7 +116,7 @@ def _set_attributes_after_config_load(self) -> None: def _try_set_rclone_password(self): # TODO: BETTER NAME! """""" input_ = input( - f"Your SSH key will be stored in the rclone config at:\n " + f"Your SSH key will be stored in the rclone config at:\n " ## TODO: FIX T HIS IS NOT SSH f"{rclone.get_full_config_filepath(self.cfg)}.\n\n" f"Would you like to set a password using Windows credential manager? " f"Press 'y' to set password or leave blank to skip." @@ -887,6 +887,9 @@ def _transfer_specific_file_or_folder( # SSH # ------------------------------------------------------------------------- + # TODO: MAKE MORE NOTES ON HOW THE GDRIVE WORKER IS THE BEST MODEL + # IT MUST BE DONE WILL NOT WORK WITHOUT + @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: @@ -938,6 +941,9 @@ def setup_ssh_connection(self) -> None: # this can just be a breaking change, but will have to handle error nicely # We could just move it from the config file, then show a warning + # TODO: need the cancel button on tui in case we close the google window + # THEN we can hide it while we make the connection to check + @check_configs_set def setup_gdrive_connection(self) -> None: """Set up a connection to Google Drive using the provided credentials. @@ -980,12 +986,16 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) - self._try_set_rclone_password() - rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( self.cfg, process, log=True ) + # If re-running connection when password already set, we don't want to + # try and set a new password + if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: + self._try_set_rclone_password() + + print("Now trying to connect or something") # TODO rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) utils.log_and_message("Google Drive Connection Successful.") diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 5b390b396..a8e8fbf11 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -115,6 +115,7 @@ def call_rclone_through_script_for_central_connection( def call_rclone_with_popen_for_central_connection( + cfg, command: str, ) -> subprocess.Popen: """Call rclone using `subprocess.Popen` for control over process termination. @@ -126,9 +127,13 @@ def call_rclone_with_popen_for_central_connection( process explicitly. """ command = "rclone " + command - process = subprocess.Popen( + lambda_func = lambda: subprocess.Popen( shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + + process = run_function_that_may_require_central_connection_password( + cfg, lambda_func + ) return process @@ -342,6 +347,7 @@ def setup_rclone_config_for_gdrive( ) process = call_rclone_with_popen_for_central_connection( + cfg, f"config create " f"{rclone_config_name} " f"drive " @@ -350,7 +356,7 @@ def setup_rclone_config_for_gdrive( f"scope drive " f"root_folder_id {cfg['gdrive_root_folder_id']} " f"{extra_args} " - f"{get_config_arg(cfg)}" + f"{get_config_arg(cfg)}", ) return process From 18158b94f8ed87993c7acfb39c91581256b0c192 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 1 Oct 2025 14:29:49 +0100 Subject: [PATCH 007/100] Working on aws. --- datashuttle/datashuttle_class.py | 8 ++++++++ datashuttle/utils/rclone.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index e5fe4b105..f440d6c2a 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1026,6 +1026,14 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) + # If re-running connection when password already set, we don't want to + # try and set a new password + if not self.cfg.rclone_has_password[ + self.cfg["connection_method"] + ]: # TODO: do this for ssh too! + self._try_set_rclone_password() + + print("Say something like checking connection...") # TODO rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) aws.raise_if_bucket_absent(self.cfg) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a8e8fbf11..52c40f18f 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -362,7 +362,7 @@ def setup_rclone_config_for_gdrive( return process -def setup_rclone_config_for_aws( +def setup_rclone_config_for_aws( # TODO: call_rclone_for_central_connection for ssh setup cfg: Configs, rclone_config_name: str, aws_secret_access_key: str, @@ -397,7 +397,8 @@ def setup_rclone_config_for_aws( else f" location_constraint {aws_region}" ) - output = call_rclone( + output = call_rclone_for_central_connection( + cfg, "config create " f"{rclone_config_name} " "s3 provider AWS " From 0bbfeca5e6be0742f3ff99888f6e0941bad1ec74 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 1 Oct 2025 17:51:41 +0100 Subject: [PATCH 008/100] Done adding to Linux. --- datashuttle/datashuttle_class.py | 5 +- datashuttle/utils/rclone.py | 11 +++-- datashuttle/utils/rclone_password.py | 68 +++++++++++++++++++-------- datashuttle/utils/test_file.xml | Bin 2027 -> 0 bytes 4 files changed, 59 insertions(+), 25 deletions(-) delete mode 100644 datashuttle/utils/test_file.xml diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index f440d6c2a..44f6d934a 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -920,7 +920,9 @@ def setup_ssh_connection(self) -> None: self._setup_rclone_central_ssh_config(private_key_str, log=True) - print("Checking write permissions on the `central_path`...") + self._try_set_rclone_password() + + print("Checking write permissions on the `central_path`...") # TODO rclone.check_successful_connection_and_raise_error_on_fail( self.cfg @@ -1692,7 +1694,6 @@ def _setup_rclone_central_ssh_config( private_key_str, log=log, ) - self._try_set_rclone_password() def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 52c40f18f..a199262b2 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -269,9 +269,14 @@ def setup_rclone_config_for_ssh( def get_config_path(): """TODO PLACEHOLDER.""" - return ( - Path().home() / "AppData" / "Roaming" / "rclone" - ) # # "$HOME/.config/rclone/rclone.conf") + if platform.system() == "Windows": + return ( + Path().home() / "AppData" / "Roaming" / "rclone" + ) # # "$HOME/.config/rclone/rclone.conf") + elif platform.system() == "Linux": + return ( + Path().home() / ".config" / "rclone" + ) def get_full_config_filepath(cfg: Configs) -> Path: diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 21924b2eb..63db7bd9e 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -42,39 +42,67 @@ def save_credentials_password(password_filepath: Path): # run it subprocess.run([shell, "-NoProfile", "-Command", ps_cmd], check=True) + elif platform.system() == "Linux": + output = subprocess.run("pass --help", shell=True, capture_output=True) + + if output.returncode != 0: + + raise RuntimeError( + "`pass` is required to set password. Install e.g. sudo apt install pass." + ) + + try: + result = subprocess.run( + ["pass", "ls"], + capture_output=True, + text=True, + check=True + ) + except subprocess.CalledProcessError as e: + if "pass init" in e.stderr: + raise Exception() # re-raise unexpected errors + + breakpoint() + subprocess.run("echo $(openssl rand -base64 40) | pass insert -m rclone/config", shell=True, check=True) + def set_credentials_as_password_command(password_filepath: Path): """""" # if platform.system() == "Windows": # filepath = Path(filepath).resolve() - shell = shutil.which("powershell") - if not shell: - raise RuntimeError("powershell.exe not found in PATH") + if platform.system() == "Windows": + shell = shutil.which("powershell") + if not shell: + raise RuntimeError("powershell.exe not found in PATH") - # Escape single quotes inside PowerShell string by doubling them - # safe_path = str(filepath).replace("'", "''") + # Escape single quotes inside PowerShell string by doubling them + # safe_path = str(filepath).replace("'", "''") - cmd = ( - f'{shell} -NoProfile -Command "Write-Output (' - f"[System.Runtime.InteropServices.Marshal]::PtrToStringAuto(" - f"[System.Runtime.InteropServices.Marshal]::SecureStringToBSTR(" - f"(Import-Clixml -LiteralPath '{password_filepath.as_posix()}' ).Password)))\"" - ) - os.environ["RCLONE_PASSWORD_COMMAND"] = cmd + cmd = ( + f'{shell} -NoProfile -Command "Write-Output (' + f"[System.Runtime.InteropServices.Marshal]::PtrToStringAuto(" + f"[System.Runtime.InteropServices.Marshal]::SecureStringToBSTR(" + f"(Import-Clixml -LiteralPath '{password_filepath.as_posix()}' ).Password)))\"" + ) + os.environ["RCLONE_PASSWORD_COMMAND"] = cmd + + elif platform.system() == "Linux": + + os.environ["RCLONE_PASSWORD_COMMAND"] = "/usr/bin/pass rclone/config" def set_rclone_password(password_filepath: Path, config_filepath: Path): """""" - assert password_filepath.exists(), ( - "password file not found at point of config creation." - ) + if platform.system() == "Windows": # TODO: handle this properly, only windows uses a password file. + assert password_filepath.exists(), ( + "password file not found at point of config creation." + ) - set_credentials_as_password_command(password_filepath) + set_credentials_as_password_command(password_filepath) # TODO: OMG handle this - subprocess.run( - f"rclone config encryption set --config {config_filepath.as_posix()} " - ) + breakpoint() + subprocess.run(f"rclone config encryption set --config {config_filepath.as_posix()}", shell=True) remove_credentials_as_password_command() @@ -84,7 +112,7 @@ def remove_rclone_password(password_filepath: Path, config_filepath: Path): """""" set_credentials_as_password_command(Path(password_filepath)) subprocess.run( - rf"rclone config encryption remove --config {config_filepath.as_posix()}" + rf"rclone config encryption remove --config {config_filepath.as_posix()}", shell=True ) # TODO: HANDLE ERRORS diff --git a/datashuttle/utils/test_file.xml b/datashuttle/utils/test_file.xml deleted file mode 100644 index ef0d66c08453304822791f2221d0b411752dc772..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2027 zcmcJQ$!-%t5Qgi_Q$%?H+vAz>f7z5g((8ud$1>_!5iQ!mVNx5Ai0>qlivgy|kaB^F8ikjdYjfcd>^a<8^eQ zUCiPRANMhZ@3619+2U=-TB6zEyy3o!%_g4y#M5RCb)z3WsDktNJTBuEbOs%8p2a7$ zm+?O42|@2jD{J1w1dD&^9ds|_I(~p~pOAOr1Lz-Ex9Plq7x&xv#xC|Ld#+F?uCRSa zh6~VS%@y)BXuEOq7kXp86ff6*L1p&O+GSsp6K@yz-w=II^c5Iyk^iqePU9SUk~6wW z9e$*OZMWKdugQCoI`tV1=W9ZR?&0k2maNz0@xa+UuV3)ddbz@S2R&i6rRsX=F{94S zf#z@{JdcsjnHA+l)bQ>_k`3{r{)|zcXSMBXpes@5JC{MNz}q9jmQR zvoe>GB`^A10TJufJjV%%+7(i3>hvy>#|o`7$*q@DX+4yQ)7dw#L#M~+8eh~Uu^K#D zH7QTLd8s8XyHzKdzI`iGPCb4{*e+9EBX%R&72`R5cC|>^+9502q&uff`$38^#9x&x z+vza_W2iBNNk$Wz>C6s45-;2~4%M)r(%RgVl X({G~R$>+Yg-sk_4{2gx3>C5~ZEMXNi From 5329c5863da2d3d9fbe979f287dd7fbc46ed3297 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 1 Oct 2025 18:40:50 +0100 Subject: [PATCH 009/100] Add to macOS. --- datashuttle/datashuttle_class.py | 3 ++- datashuttle/utils/rclone.py | 2 +- datashuttle/utils/rclone_password.py | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 44f6d934a..c8a829dac 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -920,7 +920,8 @@ def setup_ssh_connection(self) -> None: self._setup_rclone_central_ssh_config(private_key_str, log=True) - self._try_set_rclone_password() + if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: + self._try_set_rclone_password() print("Checking write permissions on the `central_path`...") # TODO diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a199262b2..a77c49bab 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -273,7 +273,7 @@ def get_config_path(): return ( Path().home() / "AppData" / "Roaming" / "rclone" ) # # "$HOME/.config/rclone/rclone.conf") - elif platform.system() == "Linux": + else: # TODO HANDLE platform.system() == "Linux": return ( Path().home() / ".config" / "rclone" ) diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 63db7bd9e..591e7dfea 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -65,6 +65,11 @@ def save_credentials_password(password_filepath: Path): breakpoint() subprocess.run("echo $(openssl rand -base64 40) | pass insert -m rclone/config", shell=True, check=True) + # TODO: HANDLE ERRORS + else: + breakpoint() + subprocess.run("security add-generic-password -a rclone -s config -w $(openssl rand -base64 40) -U", shell=True, check=True) + breakpoint() def set_credentials_as_password_command(password_filepath: Path): """""" @@ -91,6 +96,10 @@ def set_credentials_as_password_command(password_filepath: Path): os.environ["RCLONE_PASSWORD_COMMAND"] = "/usr/bin/pass rclone/config" + elif platform.system() == "Darwin": + + os.environ["RCLONE_PASSWORD_COMMAND"] = "/usr/bin/security find-generic-password -a rclone -s config -w" + def set_rclone_password(password_filepath: Path, config_filepath: Path): """""" From 43725507a657a15d8c89f5dd32d6344f6c70fcff Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 1 Oct 2025 19:02:42 +0100 Subject: [PATCH 010/100] Set different names for different projects. --- datashuttle/utils/rclone_password.py | 54 ++++++++++++++++++---------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 591e7dfea..b6d9dd10e 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -46,30 +46,38 @@ def save_credentials_password(password_filepath: Path): output = subprocess.run("pass --help", shell=True, capture_output=True) if output.returncode != 0: - raise RuntimeError( "`pass` is required to set password. Install e.g. sudo apt install pass." ) try: result = subprocess.run( - ["pass", "ls"], - capture_output=True, - text=True, - check=True + ["pass", "ls"], capture_output=True, text=True, check=True ) except subprocess.CalledProcessError as e: if "pass init" in e.stderr: - raise Exception() # re-raise unexpected errors + raise Exception() # re-raise unexpected errors breakpoint() - subprocess.run("echo $(openssl rand -base64 40) | pass insert -m rclone/config", shell=True, check=True) + subprocess.run( + f"echo $(openssl rand -base64 40) | pass insert -m {name_from_file(password_filepath)}", + shell=True, + check=True, + ) # TODO: HANDLE ERRORS else: - breakpoint() - subprocess.run("security add-generic-password -a rclone -s config -w $(openssl rand -base64 40) -U", shell=True, check=True) - breakpoint() + subprocess.run( + f"security add-generic-password -a datashuttle -s {name_from_file(password_filepath)} -w $(openssl rand -base64 40) -U", + shell=True, + check=True, + ) + + +def name_from_file(password_filepath): # TODO: HADNLE THIS MUCH LESS WEIRDLY! + """""" + return f"datashuttle/rclone/{password_filepath.stem}" + def set_credentials_as_password_command(password_filepath: Path): """""" @@ -93,25 +101,34 @@ def set_credentials_as_password_command(password_filepath: Path): os.environ["RCLONE_PASSWORD_COMMAND"] = cmd elif platform.system() == "Linux": - - os.environ["RCLONE_PASSWORD_COMMAND"] = "/usr/bin/pass rclone/config" + os.environ["RCLONE_PASSWORD_COMMAND"] = ( + f"/usr/bin/pass {name_from_file(password_filepath)}" + ) elif platform.system() == "Darwin": - - os.environ["RCLONE_PASSWORD_COMMAND"] = "/usr/bin/security find-generic-password -a rclone -s config -w" + os.environ["RCLONE_PASSWORD_COMMAND"] = ( + f"/usr/bin/security find-generic-password -a datashuttle -s {name_from_file(password_filepath)} -w" + ) def set_rclone_password(password_filepath: Path, config_filepath: Path): """""" - if platform.system() == "Windows": # TODO: handle this properly, only windows uses a password file. + if ( + platform.system() == "Windows" + ): # TODO: handle this properly, only windows uses a password file. assert password_filepath.exists(), ( "password file not found at point of config creation." ) - set_credentials_as_password_command(password_filepath) # TODO: OMG handle this + set_credentials_as_password_command( + password_filepath + ) # TODO: OMG handle this breakpoint() - subprocess.run(f"rclone config encryption set --config {config_filepath.as_posix()}", shell=True) + subprocess.run( + f"rclone config encryption set --config {config_filepath.as_posix()}", + shell=True, + ) remove_credentials_as_password_command() @@ -121,7 +138,8 @@ def remove_rclone_password(password_filepath: Path, config_filepath: Path): """""" set_credentials_as_password_command(Path(password_filepath)) subprocess.run( - rf"rclone config encryption remove --config {config_filepath.as_posix()}", shell=True + rf"rclone config encryption remove --config {config_filepath.as_posix()}", + shell=True, ) # TODO: HANDLE ERRORS From 366d67b6fdc7a450679f262438d220cbff934a82 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 2 Oct 2025 18:29:50 +0100 Subject: [PATCH 011/100] Refactoring. --- datashuttle/configs/canonical_folders.py | 12 ++ datashuttle/configs/config_class.py | 10 ++ datashuttle/datashuttle_class.py | 206 +++++++++++------------ datashuttle/utils/rclone.py | 34 ++-- datashuttle/utils/rclone_password.py | 44 ++--- datashuttle/utils/utils.py | 16 +- 6 files changed, 167 insertions(+), 155 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index da4c92a65..52dee4de1 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -6,6 +6,8 @@ if TYPE_CHECKING: from datashuttle.utils.custom_types import TopLevelFolder +import platform + from datashuttle.configs import canonical_configs from datashuttle.utils.folder_class import Folder @@ -91,3 +93,13 @@ def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: temp_logs_path = base_path / "temp_logs" return base_path, temp_logs_path + + +def get_rclone_config_base_path(): + """TODO PLACEHOLDER.""" + if platform.system() == "Windows": + return ( + Path().home() / "AppData" / "Roaming" / "rclone" + ) # # "$HOME/.config/rclone/rclone.conf") + else: # TODO HANDLE platform.system() == "Linux": + return Path().home() / ".config" / "rclone" diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index b227d4667..3087e6ec6 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -70,6 +70,9 @@ def __init__( self.rclone_has_password = {} self.setup_rclone_has_password() + def connection_method_rclone_config_has_password(self): + return self.rclone_has_password[self["connection_method"]] + def setup_rclone_has_password(self): """""" if self.rclone_password_state_file_path.is_file(): @@ -288,6 +291,13 @@ def get_rclone_config_name( return f"central_{self.project_name}_{connection_method}" + def get_rclone_config_filepath(self) -> Path: + """""" + return ( + canonical_folders.get_rclone_config_base_path() + / f"{self.get_rclone_config_name()}.conf" + ) + def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index c8a829dac..ed29d4832 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -4,6 +4,7 @@ import glob import json import os +import platform import shutil from pathlib import Path from typing import ( @@ -113,92 +114,6 @@ def _set_attributes_after_config_load(self) -> None: self._make_project_metadata_if_does_not_exist() - def _try_set_rclone_password(self): # TODO: BETTER NAME! - """""" - input_ = input( - f"Your SSH key will be stored in the rclone config at:\n " ## TODO: FIX T HIS IS NOT SSH - f"{rclone.get_full_config_filepath(self.cfg)}.\n\n" - f"Would you like to set a password using Windows credential manager? " - f"Press 'y' to set password or leave blank to skip." - ) - - set_password = input_ == "y" - - if set_password: - try: - self.set_rclone_password() - except BaseException as e: - print(e) - # THIS PATH IS WRONG - config_path = rclone.get_full_config_filepath(self.cfg) - - raise RuntimeError( - f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" - f"Use set_rclone_password()` to attempt to set the password again (see stacktrace above). " - ) - - print("Password set successfully") - - def set_rclone_password(self): - """""" - # TODO: CHECK CONNECTION METHOD - connection_method = self.cfg["connection_method"] - - if self.cfg.rclone_has_password[connection_method]: - raise RuntimeError( - "This config file already has a password set. First, use `remove_rclone_password` to remove it." - ) - - rclone_config_path = rclone.get_full_config_filepath( - self.cfg - ) # change name to rclone config because this is getting confusing! - - if not rclone_config_path.exists(): - raise RuntimeError( - f"Rclone config file for: {connection_method} was not found. " - f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" - ) - - password_filepath = rclone_password.get_password_filepath(self.cfg) - - if password_filepath.exists(): - password_filepath.unlink() - - rclone_password.save_credentials_password( - password_filepath, - ) - - rclone_password.set_rclone_password( - password_filepath, rclone.get_full_config_filepath(self.cfg) - ) - - self.cfg.rclone_has_password[connection_method] = ( - True # HANDLE THIS PROPERLY - ) - self.cfg.save_rclone_password_state() - print(self.cfg.rclone_has_password[connection_method]) - - def remove_rclone_password(self): - """""" - # TODO: CHECK CONNECTION METHOD - connection_method = self.cfg["connection_method"] - - if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: - raise RuntimeError( - f"The config for the current connection method: {self.cfg['connection_method']} does not have a password. Cannot remove." - ) - config_filepath = rclone_password.get_password_filepath(self.cfg) - rclone_password.remove_rclone_password( - config_filepath, rclone.get_full_config_filepath(self.cfg) - ) - - self.cfg.rclone_has_password[connection_method] = ( - False # HANDLE THIS PROPERLY - ) - self.cfg.save_rclone_password_state() - - print("SAY SOMETHING LIKE PASSWORD REMOVED") - # ------------------------------------------------------------------------- # Public Folder Makers # ------------------------------------------------------------------------- @@ -890,6 +805,14 @@ def _transfer_specific_file_or_folder( # TODO: MAKE MORE NOTES ON HOW THE GDRIVE WORKER IS THE BEST MODEL # IT MUST BE DONE WILL NOT WORK WITHOUT + # TODO: this is going to be a massive pain because old config files will not work + # will need to re-set up all connections + # this can just be a breaking change, but will have to handle error nicely + # We could just move it from the config file, then show a warning + + # TODO: need the cancel button on tui in case we close the google window + # THEN we can hide it while we make the connection to check + @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: @@ -920,10 +843,13 @@ def setup_ssh_connection(self) -> None: self._setup_rclone_central_ssh_config(private_key_str, log=True) - if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: - self._try_set_rclone_password() + utils.log_and_message( + f"Your SSH key will be stored in the rclone config at:\n " + f"{self.cfg.get_rclone_config_filepath()}.\n\n" + ) - print("Checking write permissions on the `central_path`...") # TODO + if not self.cfg.connection_method_rclone_config_has_password(): + self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail( self.cfg @@ -939,14 +865,6 @@ def setup_ssh_connection(self) -> None: # Google Drive # ------------------------------------------------------------------------- - # TODO: this is going to be a massive pain because old config files will not work - # will need to re-set up all connections - # this can just be a breaking change, but will have to handle error nicely - # We could just move it from the config file, then show a warning - - # TODO: need the cancel button on tui in case we close the google window - # THEN we can hide it while we make the connection to check - @check_configs_set def setup_gdrive_connection(self) -> None: """Set up a connection to Google Drive using the provided credentials. @@ -993,12 +911,9 @@ def setup_gdrive_connection(self) -> None: self.cfg, process, log=True ) - # If re-running connection when password already set, we don't want to - # try and set a new password - if not self.cfg.rclone_has_password[self.cfg["connection_method"]]: + if not self.cfg.connection_method_rclone_config_has_password(): self._try_set_rclone_password() - print("Now trying to connect or something") # TODO rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) utils.log_and_message("Google Drive Connection Successful.") @@ -1029,14 +944,9 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) - # If re-running connection when password already set, we don't want to - # try and set a new password - if not self.cfg.rclone_has_password[ - self.cfg["connection_method"] - ]: # TODO: do this for ssh too! + if not self.cfg.connection_method_rclone_config_has_password(): self._try_set_rclone_password() - print("Say something like checking connection...") # TODO rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) aws.raise_if_bucket_absent(self.cfg) @@ -1044,6 +954,86 @@ def setup_aws_connection(self) -> None: ds_logger.close_log_filehandler() + # ------------------------------------------------------------------------- + # Rclone config password + # ------------------------------------------------------------------------- + + def _try_set_rclone_password(self): + """""" + pass_type = { + "Windows": "Windows credential manager", + "Linux": "the `pass` program", + "Darwin": "macOS inbuild `security`." + } + + input_ = utils.get_user_input( + f"Would you like to set a password using {pass_type[platform.system()]}.\n" + f"Press 'y' to set password or leave blank to skip." + ) + + set_password = input_ == "y" + + if set_password: + try: + self.set_rclone_password() + except Exception as e: + + config_path = self.cfg.get_rclone_config_filepath() + + utils.log_and_raise_error( + f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" + f"Use set_rclone_password()` to attempt to set the password again (see full error message above). ", + RuntimeError, + from_error=e + ) + + utils.log_and_message("Password set successfully") + + def set_rclone_password(self): + """""" + if self.cfg.connection_method_rclone_config_has_password(): + raise RuntimeError( + "This config file already has a password set. " + "First, use `remove_rclone_password` to remove it." + ) + + connection_method = self.cfg["connection_method"] + + rclone_config_path = self.cfg.get_rclone_config_filepath() + + if not rclone_config_path.exists(): + raise RuntimeError( + f"Rclone config file for: {connection_method} was not found. " + f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" + ) + rclone_password.run_rclone_config_encrypt( + self.cfg + ) + + self.cfg.rclone_has_password[connection_method] = True + + self.cfg.save_rclone_password_state() + + def remove_rclone_password(self): + """""" + if not self.cfg.connection_method_rclone_config_has_password(): + raise RuntimeError( + f"The config for the current connection method: {self.cfg['connection_method']} does not have a password. Cannot remove." + ) + config_filepath = rclone_password.get_password_filepath(self.cfg) + + rclone_password.remove_rclone_password( + config_filepath, self.cfg.get_rclone_config_filepath() + ) + + self.cfg.rclone_has_password[self.cfg["connection_method"]] = False + + self.cfg.save_rclone_password_state() + + utils.log_and_message( + f"Password removed from rclone config file: {}" + ) + # ------------------------------------------------------------------------- # Configs # ------------------------------------------------------------------------- @@ -1062,7 +1052,7 @@ def make_config_file( ) -> None: """Initialize the configurations for datashuttle on the local machine. - Once initialised, these settings will be used each + Once initialized, these settings will be used each time the datashuttle is opened. These settings are stored in a config file on the diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a77c49bab..dd5543b2d 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -11,7 +11,6 @@ import shlex import subprocess import tempfile -from pathlib import Path from subprocess import CompletedProcess from datashuttle.configs import canonical_configs @@ -245,9 +244,8 @@ def setup_rclone_config_for_ssh( """ key_escaped = private_key_str.replace("\n", "\\n") - rclone_config_filepath = get_full_config_filepath( - cfg - ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file + rclone_config_filepath = cfg.get_rclone_config_filepath() + # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() @@ -267,28 +265,12 @@ def setup_rclone_config_for_ssh( log_rclone_config_output() -def get_config_path(): - """TODO PLACEHOLDER.""" - if platform.system() == "Windows": - return ( - Path().home() / "AppData" / "Roaming" / "rclone" - ) # # "$HOME/.config/rclone/rclone.conf") - else: # TODO HANDLE platform.system() == "Linux": - return ( - Path().home() / ".config" / "rclone" - ) - - -def get_full_config_filepath(cfg: Configs) -> Path: - return get_config_path() / f"{cfg.get_rclone_config_name()}.conf" - - def get_config_arg(cfg): """TODO PLACEHOLDER.""" cfg.get_rclone_config_name() # pass this? handle better... if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: - return f'--config "{get_full_config_filepath(cfg)}"' + return f'--config "{cfg.get_rclone_config_filepath()}"' else: return "" @@ -430,8 +412,14 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: If the command fails, it raises a ConnectionError. The created file is deleted thereafter. """ + utils.log_and_message( + f"Attempting to write to the central server to check write permissions...\n" + f"`central_path`: {cfg['central_path']}" + ) + filename = f"{utils.get_random_string()}_temp.txt" + # Get the full path to write to if cfg["central_path"] is None: assert cfg["connection_method"] == "gdrive", ( "`central_path` may only be `None` for `gdrive`" @@ -440,6 +428,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: else: tempfile_path = (cfg["central_path"] / filename).as_posix() + # Try and write to the central location and log errors config_name = cfg.get_rclone_config_name() output = call_rclone_for_central_connection( @@ -452,6 +441,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output.stderr.decode("utf-8"), ConnectionError ) + # Delete the written file and complete output = call_rclone_for_central_connection( cfg, f"delete {cfg.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", @@ -462,6 +452,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output.stderr.decode("utf-8"), ConnectionError ) + utils.log_and_message("Successfully wrote to the central location.") + def log_rclone_config_output() -> None: # TODO: remove or update this """Log the output from creating Rclone config.""" diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index b6d9dd10e..a8df4b07c 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -4,6 +4,8 @@ import subprocess from pathlib import Path +from configs.config_class import Configs + from datashuttle.configs import canonical_folders @@ -22,9 +24,14 @@ def get_password_filepath( return base_path / f"{cfg.get_rclone_config_name()}.xml" -def save_credentials_password(password_filepath: Path): +def save_credentials_password(cfg): """""" if platform.system() == "Windows": + password_filepath = get_password_filepath(cfg) + + if password_filepath.exists(): + password_filepath.unlink() + # $env:APPDATA\\rclone\\rclone-credential.xml shell = shutil.which("powershell") if not shell: @@ -60,7 +67,7 @@ def save_credentials_password(password_filepath: Path): breakpoint() subprocess.run( - f"echo $(openssl rand -base64 40) | pass insert -m {name_from_file(password_filepath)}", + f"echo $(openssl rand -base64 40) | pass insert -m {cfg.get_rclone_config_name()}", shell=True, check=True, ) @@ -68,7 +75,7 @@ def save_credentials_password(password_filepath: Path): # TODO: HANDLE ERRORS else: subprocess.run( - f"security add-generic-password -a datashuttle -s {name_from_file(password_filepath)} -w $(openssl rand -base64 40) -U", + f"security add-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", shell=True, check=True, ) @@ -79,12 +86,15 @@ def name_from_file(password_filepath): # TODO: HADNLE THIS MUCH LESS WEIRDLY! return f"datashuttle/rclone/{password_filepath.stem}" -def set_credentials_as_password_command(password_filepath: Path): +def set_credentials_as_password_command(cfg): """""" - # if platform.system() == "Windows": - # filepath = Path(filepath).resolve() - if platform.system() == "Windows": + password_filepath = get_password_filepath(cfg) + + assert password_filepath.exists(), ( + "Critical error: password file not found when setting password command." + ) + shell = shutil.which("powershell") if not shell: raise RuntimeError("powershell.exe not found in PATH") @@ -96,35 +106,25 @@ def set_credentials_as_password_command(password_filepath: Path): f'{shell} -NoProfile -Command "Write-Output (' f"[System.Runtime.InteropServices.Marshal]::PtrToStringAuto(" f"[System.Runtime.InteropServices.Marshal]::SecureStringToBSTR(" - f"(Import-Clixml -LiteralPath '{password_filepath.as_posix()}' ).Password)))\"" + f"(Import-Clixml -LiteralPath '{password_filepath}' ).Password)))\"" ) os.environ["RCLONE_PASSWORD_COMMAND"] = cmd elif platform.system() == "Linux": os.environ["RCLONE_PASSWORD_COMMAND"] = ( - f"/usr/bin/pass {name_from_file(password_filepath)}" + f"/usr/bin/pass {cfg.get_rclone_config_name()}" ) elif platform.system() == "Darwin": os.environ["RCLONE_PASSWORD_COMMAND"] = ( - f"/usr/bin/security find-generic-password -a datashuttle -s {name_from_file(password_filepath)} -w" + f"/usr/bin/security find-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w" ) -def set_rclone_password(password_filepath: Path, config_filepath: Path): +def run_rclone_config_encrypt(cfg: Configs): """""" - if ( - platform.system() == "Windows" - ): # TODO: handle this properly, only windows uses a password file. - assert password_filepath.exists(), ( - "password file not found at point of config creation." - ) - - set_credentials_as_password_command( - password_filepath - ) # TODO: OMG handle this + set_credentials_as_password_command(cfg) - breakpoint() subprocess.run( f"rclone config encryption set --config {config_filepath.as_posix()}", shell=True, diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index ea8c279bf..967a86038 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -45,13 +45,15 @@ def log_and_message(message: str, use_rich: bool = False) -> None: print_message_to_user(message, use_rich) -def log_and_raise_error(message: str, exception: Any) -> None: +def log_and_raise_error( + message: str, exception: Any, from_error: Exception | None = None +) -> None: """Log the message before raising the same message as an error.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.error(f"\n\n{' '.join(traceback.format_stack(limit=5))}") logger.error(message) - raise_error(message, exception) + raise_error(message, exception, from_error=from_error) def warn(message: str, log: bool) -> None: @@ -72,14 +74,20 @@ def warn(message: str, log: bool) -> None: warnings.warn(message) -def raise_error(message: str, exception) -> None: +def raise_error( + message: str, exception, from_error: Exception | None = None +) -> None: """Centralized way to raise an error. The logger is closed to ensure it is not still running if a function call raises an exception in a python environment. """ ds_logger.close_log_filehandler() - raise exception(message) + + if from_error: + raise exception(message) from from_error + else: + raise exception(message) def print_message_to_user( From ebf073a570fb3806019aaa2c151df1aea84ce1dc Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 2 Oct 2025 18:58:48 +0100 Subject: [PATCH 012/100] Refactoring more. --- datashuttle/datashuttle_class.py | 32 ++++++--------------------- datashuttle/utils/rclone_password.py | 33 ++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index ed29d4832..a9e870678 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -963,7 +963,7 @@ def _try_set_rclone_password(self): pass_type = { "Windows": "Windows credential manager", "Linux": "the `pass` program", - "Darwin": "macOS inbuild `security`." + "Darwin": "macOS inbuild `security`.", } input_ = utils.get_user_input( @@ -977,14 +977,13 @@ def _try_set_rclone_password(self): try: self.set_rclone_password() except Exception as e: - config_path = self.cfg.get_rclone_config_filepath() utils.log_and_raise_error( f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" f"Use set_rclone_password()` to attempt to set the password again (see full error message above). ", RuntimeError, - from_error=e + from_error=e, ) utils.log_and_message("Password set successfully") @@ -997,20 +996,9 @@ def set_rclone_password(self): "First, use `remove_rclone_password` to remove it." ) - connection_method = self.cfg["connection_method"] - - rclone_config_path = self.cfg.get_rclone_config_filepath() - - if not rclone_config_path.exists(): - raise RuntimeError( - f"Rclone config file for: {connection_method} was not found. " - f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" - ) - rclone_password.run_rclone_config_encrypt( - self.cfg - ) + rclone_password.run_rclone_config_encrypt(self.cfg) - self.cfg.rclone_has_password[connection_method] = True + self.cfg.rclone_has_password[self.cfg["connection_method"]] = True self.cfg.save_rclone_password_state() @@ -1018,22 +1006,16 @@ def remove_rclone_password(self): """""" if not self.cfg.connection_method_rclone_config_has_password(): raise RuntimeError( - f"The config for the current connection method: {self.cfg['connection_method']} does not have a password. Cannot remove." + f"The config for the current connection method: {self.cfg['connection_method']} " + f"does not have a password. Cannot remove." ) - config_filepath = rclone_password.get_password_filepath(self.cfg) - rclone_password.remove_rclone_password( - config_filepath, self.cfg.get_rclone_config_filepath() - ) + rclone_password.remove_rclone_password(self.cfg) self.cfg.rclone_has_password[self.cfg["connection_method"]] = False self.cfg.save_rclone_password_state() - utils.log_and_message( - f"Password removed from rclone config file: {}" - ) - # ------------------------------------------------------------------------- # Configs # ------------------------------------------------------------------------- diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index a8df4b07c..02fb4e0a5 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -7,6 +7,7 @@ from configs.config_class import Configs from datashuttle.configs import canonical_folders +from datashuttle.utils import utils def get_password_filepath( @@ -47,6 +48,7 @@ def save_credentials_password(cfg): ) # run it + # TODO: HANDLE ERRORS subprocess.run([shell, "-NoProfile", "-Command", ps_cmd], check=True) elif platform.system() == "Linux": @@ -58,6 +60,7 @@ def save_credentials_password(cfg): ) try: + # TODO: HANDLE ERRORS result = subprocess.run( ["pass", "ls"], capture_output=True, text=True, check=True ) @@ -66,6 +69,7 @@ def save_credentials_password(cfg): raise Exception() # re-raise unexpected errors breakpoint() + # TODO: HANDLE ERRORS subprocess.run( f"echo $(openssl rand -base64 40) | pass insert -m {cfg.get_rclone_config_name()}", shell=True, @@ -74,6 +78,7 @@ def save_credentials_password(cfg): # TODO: HANDLE ERRORS else: + # TODO: HANDLE ERRORS subprocess.run( f"security add-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", shell=True, @@ -123,10 +128,23 @@ def set_credentials_as_password_command(cfg): def run_rclone_config_encrypt(cfg: Configs): """""" + rclone_config_path = cfg.get_rclone_config_filepath() + + if not rclone_config_path.exists(): + connection_method = cfg["connection_method"] + + raise RuntimeError( + f"Rclone config file for: {connection_method} was not found. " + f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" + ) + + save_credentials_password(cfg) + set_credentials_as_password_command(cfg) + # TODO: HANDLE ERRORS subprocess.run( - f"rclone config encryption set --config {config_filepath.as_posix()}", + f"rclone config encryption set --config {rclone_config_path.as_posix()}", shell=True, ) @@ -134,17 +152,24 @@ def run_rclone_config_encrypt(cfg: Configs): # TODO: HANDLE ERRORS -def remove_rclone_password(password_filepath: Path, config_filepath: Path): +def remove_rclone_password(cfg): """""" - set_credentials_as_password_command(Path(password_filepath)) + set_credentials_as_password_command(Path(cfg)) + + config_filepath = cfg.get_rclone_config_filepath() + + # TODO: HANDLE ERRORS subprocess.run( rf"rclone config encryption remove --config {config_filepath.as_posix()}", shell=True, ) - # TODO: HANDLE ERRORS remove_credentials_as_password_command() + utils.log_and_message( + f"Password removed from rclone config file: {config_filepath}" + ) + def remove_credentials_as_password_command(): if "RCLONE_PASSWORD_COMMAND" in os.environ: From 53dcc5ca80eb829a153c0b61dc40fe7c4d6a2fe3 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 3 Oct 2025 16:07:47 +0100 Subject: [PATCH 013/100] Tidy up rclone_password.py --- datashuttle/utils/rclone.py | 55 ++++--- datashuttle/utils/rclone_password.py | 212 ++++++++++++++++++--------- 2 files changed, 167 insertions(+), 100 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index dd5543b2d..c1addfa94 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -11,6 +11,7 @@ import shlex import subprocess import tempfile +from pathlib import Path from subprocess import CompletedProcess from datashuttle.configs import canonical_configs @@ -114,7 +115,6 @@ def call_rclone_through_script_for_central_connection( def call_rclone_with_popen_for_central_connection( - cfg, command: str, ) -> subprocess.Popen: """Call rclone using `subprocess.Popen` for control over process termination. @@ -126,18 +126,14 @@ def call_rclone_with_popen_for_central_connection( process explicitly. """ command = "rclone " + command - lambda_func = lambda: subprocess.Popen( + process = subprocess.Popen( shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - - process = run_function_that_may_require_central_connection_password( - cfg, lambda_func - ) return process def await_call_rclone_with_popen_for_central_connection_raise_on_fail( - cfg, process: subprocess.Popen, log: bool = True + process: subprocess.Popen, log: bool = True ): """Await rclone the subprocess.Popen call. @@ -161,11 +157,10 @@ def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ - set_password = cfg.rclone_has_password[cfg["connection_method"]] + set_password = cfg.connection_method_rclone_config_has_password() if set_password: - config_filepath = rclone_password.get_password_filepath(cfg) - rclone_password.set_credentials_as_password_command(config_filepath) + rclone_password.set_credentials_as_password_command(cfg) results = lambda_func() @@ -244,8 +239,9 @@ def setup_rclone_config_for_ssh( """ key_escaped = private_key_str.replace("\n", "\\n") - rclone_config_filepath = cfg.get_rclone_config_filepath() - # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file + rclone_config_filepath = get_full_config_filepath( + cfg + ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() @@ -265,12 +261,23 @@ def setup_rclone_config_for_ssh( log_rclone_config_output() +def get_config_path(): + """TODO PLACEHOLDER.""" + return ( + Path().home() / "AppData" / "Roaming" / "rclone" + ) # # "$HOME/.config/rclone/rclone.conf") + + +def get_full_config_filepath(cfg: Configs) -> Path: + return get_config_path() / f"{cfg.get_rclone_config_name()}.conf" + + def get_config_arg(cfg): """TODO PLACEHOLDER.""" cfg.get_rclone_config_name() # pass this? handle better... if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: - return f'--config "{cfg.get_rclone_config_filepath()}"' + return f'--config "{get_full_config_filepath(cfg)}"' else: return "" @@ -334,7 +341,6 @@ def setup_rclone_config_for_gdrive( ) process = call_rclone_with_popen_for_central_connection( - cfg, f"config create " f"{rclone_config_name} " f"drive " @@ -343,13 +349,13 @@ def setup_rclone_config_for_gdrive( f"scope drive " f"root_folder_id {cfg['gdrive_root_folder_id']} " f"{extra_args} " - f"{get_config_arg(cfg)}", + f"{get_config_arg(cfg)}" ) return process -def setup_rclone_config_for_aws( # TODO: call_rclone_for_central_connection for ssh setup +def setup_rclone_config_for_aws( cfg: Configs, rclone_config_name: str, aws_secret_access_key: str, @@ -384,8 +390,7 @@ def setup_rclone_config_for_aws( # TODO: call_rclone_for_central_connection for else f" location_constraint {aws_region}" ) - output = call_rclone_for_central_connection( - cfg, + output = call_rclone( "config create " f"{rclone_config_name} " "s3 provider AWS " @@ -412,14 +417,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: If the command fails, it raises a ConnectionError. The created file is deleted thereafter. """ - utils.log_and_message( - f"Attempting to write to the central server to check write permissions...\n" - f"`central_path`: {cfg['central_path']}" - ) - filename = f"{utils.get_random_string()}_temp.txt" - # Get the full path to write to if cfg["central_path"] is None: assert cfg["connection_method"] == "gdrive", ( "`central_path` may only be `None` for `gdrive`" @@ -428,7 +427,6 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: else: tempfile_path = (cfg["central_path"] / filename).as_posix() - # Try and write to the central location and log errors config_name = cfg.get_rclone_config_name() output = call_rclone_for_central_connection( @@ -441,7 +439,6 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output.stderr.decode("utf-8"), ConnectionError ) - # Delete the written file and complete output = call_rclone_for_central_connection( cfg, f"delete {cfg.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", @@ -452,8 +449,6 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output.stderr.decode("utf-8"), ConnectionError ) - utils.log_and_message("Successfully wrote to the central location.") - def log_rclone_config_output() -> None: # TODO: remove or update this """Log the output from creating Rclone config.""" @@ -544,7 +539,7 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) - # if cfg.rclone_has_password[cfg["connection_method"]]: # TODO: one getter + # if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter # print("SET") # config_filepath = rclone_password.get_password_filepath( # cfg @@ -567,7 +562,7 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.rclone_has_password[cfg["connection_method"]]: + if cfg.connection_method_rclone_config_has_password(): print("REMOVED") rclone_password.remove_credentials_as_password_command() diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 02fb4e0a5..fabebfb40 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -1,94 +1,123 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from datashuttle.configs.configs_class import Configs + import os import platform import shutil import subprocess -from pathlib import Path - -from configs.config_class import Configs from datashuttle.configs import canonical_folders from datashuttle.utils import utils -def get_password_filepath( - cfg, -): # Configs # TOOD: datashuttle_path should be on configs? - """""" - assert cfg["connection_method"] in ["aws", "gdrive", "ssh"], ( - "password should only be set for ssh, aws, gdrive." - ) - - base_path = canonical_folders.get_datashuttle_path() / "credentials" - - base_path.mkdir(exist_ok=True, parents=True) - - return base_path / f"{cfg.get_rclone_config_name()}.xml" - - def save_credentials_password(cfg): """""" if platform.system() == "Windows": - password_filepath = get_password_filepath(cfg) + set_password_windows(cfg) + elif platform.system() == "Linux": + set_password_linux(cfg) + else: + set_password_macos(cfg) - if password_filepath.exists(): - password_filepath.unlink() - # $env:APPDATA\\rclone\\rclone-credential.xml - shell = shutil.which("powershell") - if not shell: - raise RuntimeError( - "powershell.exe not found in PATH (need Windows PowerShell 5.1)." - ) +def set_password_windows(cfg: Configs): + """""" + password_filepath = get_password_filepath(cfg) + + if password_filepath.exists(): + password_filepath.unlink() - ps_cmd = ( - "Add-Type -AssemblyName System.Web; " - "New-Object PSCredential 'rclone', " - "(ConvertTo-SecureString ([System.Web.Security.Membership]::GeneratePassword(40,10)) -AsPlainText -Force) " - f"| Export-Clixml -LiteralPath '{password_filepath}'" + shell = shutil.which("powershell") + if not shell: + raise RuntimeError( + "powershell.exe not found in PATH (need Windows PowerShell 5.1)." ) - # run it - # TODO: HANDLE ERRORS - subprocess.run([shell, "-NoProfile", "-Command", ps_cmd], check=True) + ps_cmd = ( + "Add-Type -AssemblyName System.Web; " + "New-Object PSCredential 'rclone', " + "(ConvertTo-SecureString ([System.Web.Security.Membership]::GeneratePassword(40,10)) -AsPlainText -Force) " + f"| Export-Clixml -LiteralPath '{password_filepath}'" + ) + output = subprocess.run( + [shell, "-NoProfile", "-Command", ps_cmd], + capture_output=True, + text=True, + ) + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}", + f"\n--- STDERR ---\n{output.stderr}", + "Could not set the PSCredential with System.web. See the error message above.", + ) - elif platform.system() == "Linux": - output = subprocess.run("pass --help", shell=True, capture_output=True) - if output.returncode != 0: - raise RuntimeError( - "`pass` is required to set password. Install e.g. sudo apt install pass." - ) +def set_password_linux(cfg): + """""" + output = subprocess.run( + "pass --help", + shell=True, + capture_output=True, + text=True, + ) + if output.returncode != 0: + raise RuntimeError( + "`pass` is required to set password. Install e.g. sudo apt install pass." + ) - try: - # TODO: HANDLE ERRORS - result = subprocess.run( - ["pass", "ls"], capture_output=True, text=True, check=True - ) - except subprocess.CalledProcessError as e: - if "pass init" in e.stderr: - raise Exception() # re-raise unexpected errors - - breakpoint() - # TODO: HANDLE ERRORS - subprocess.run( - f"echo $(openssl rand -base64 40) | pass insert -m {cfg.get_rclone_config_name()}", + try: + output = subprocess.run( + ["pass", "ls"], shell=True, - check=True, + capture_output=True, + text=True, ) + except subprocess.CalledProcessError as e: + if "pass init" in e.stderr: + raise RuntimeError( + "Password store is not initialized. " + "Run `pass init ` before using `pass`." + ) from e + else: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}", + f"\n--- STDERR ---\n{output.stderr}", + "Could not set up password with `pass`. See the error message above.", + ) - # TODO: HANDLE ERRORS - else: - # TODO: HANDLE ERRORS - subprocess.run( - f"security add-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", - shell=True, - check=True, + output = subprocess.run( + f"echo $(openssl rand -base64 40) | pass insert -m {cfg.get_rclone_config_name()}", + shell=True, + capture_output=True, + text=True, + ) + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}", + f"\n--- STDERR ---\n{output.stderr}", + "Could not remove the password from the RClone config. See the error message above.", ) -def name_from_file(password_filepath): # TODO: HADNLE THIS MUCH LESS WEIRDLY! +def set_password_macos(cfg: Configs): """""" - return f"datashuttle/rclone/{password_filepath.stem}" + output = subprocess.run( + f"security add-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", + shell=True, + capture_output=True, + text=True, + ) + + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}", + f"\n--- STDERR ---\n{output.stderr}", + "Could not remove the password from the RClone config. See the error message above.", + ) def set_credentials_as_password_command(cfg): @@ -142,27 +171,40 @@ def run_rclone_config_encrypt(cfg: Configs): set_credentials_as_password_command(cfg) - # TODO: HANDLE ERRORS - subprocess.run( + output = subprocess.run( f"rclone config encryption set --config {rclone_config_path.as_posix()}", shell=True, + capture_output=True, + text=True, ) + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}\n" + f"\n--- STDERR ---\n{output.stderr}\n" + "Could not remove the password from the RClone config. See the error message above." + ) remove_credentials_as_password_command() -# TODO: HANDLE ERRORS def remove_rclone_password(cfg): """""" - set_credentials_as_password_command(Path(cfg)) + set_credentials_as_password_command(cfg) config_filepath = cfg.get_rclone_config_filepath() - # TODO: HANDLE ERRORS - subprocess.run( + output = subprocess.run( rf"rclone config encryption remove --config {config_filepath.as_posix()}", shell=True, + capture_output=True, + text=True, ) + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}", + f"\n--- STDERR ---\n{output.stderr}", + "Could not remove the password from the RClone config. See the error message above.", + ) remove_credentials_as_password_command() @@ -174,3 +216,33 @@ def remove_rclone_password(cfg): def remove_credentials_as_password_command(): if "RCLONE_PASSWORD_COMMAND" in os.environ: os.environ.pop("RCLONE_PASSWORD_COMMAND") + + +def get_password_filepath( + cfg, +): # Configs # TODO: datashuttle_path should be on configs? + """""" + assert cfg["connection_method"] in ["aws", "gdrive", "ssh"], ( + "password should only be set for ssh, aws, gdrive." + ) + + base_path = canonical_folders.get_datashuttle_path() / "credentials" + + base_path.mkdir(exist_ok=True, parents=True) + + return base_path / f"{cfg.get_rclone_config_name()}.xml" + + +def run_raise_if_fail(command, command_description): + output = run_subprocess.run( + command, + shell=True, # TODO: handle shell + capture_output=True, + text=True, + ) + + if output.returncode != 0: + raise RuntimeError( + f"\n--- STDOUT ---\n{output.stdout}\n" + f"\n--- STDERR ---\n{output.stderr}\n" + ) From 22a8db0e67a2a84562b472f01ed6e8ae41654018 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sat, 4 Oct 2025 08:41:34 +0100 Subject: [PATCH 014/100] Editing setup_ssh. --- datashuttle/tui/interface.py | 10 +++++ datashuttle/tui/screens/setup_ssh.py | 64 +++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 00f6e0697..6d50cebf7 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -625,3 +625,13 @@ def setup_aws_connection( return True, None except BaseException as e: return False, str(e) + + # Set RClone Password + # ------------------------------------------------------------------------------------ + + def try_setup_rclone_password(self): + try: + self.project._try_set_password() + return True, None + except BaseException as e: + return False, str(e) diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 3ab5025e8..7802e474e 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -67,19 +67,28 @@ def on_button_pressed(self, event: Button.pressed) -> None: input, multiple attempts are allowed. """ if event.button.id == "setup_ssh_cancel_button": - self.dismiss() + if self.stage == 3: + self.show_connection_sucesssful_message() + else: + self.dismiss() if event.button.id == "setup_ssh_ok_button": if self.stage == 0: self.ask_user_to_accept_hostkeys() - elif self.stage == 1: + elif self.stage == 1: # TODO: use str for stages self.save_hostkeys_and_prompt_password_input() elif self.stage == 2: self.use_password_to_setup_ssh_key_pairs() elif self.stage == 3: + self.ask_setup_rclone_password() + + elif self.stage == 4: + self.show_connection_sucesssful_message() + + elif self.stage == 5: self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: @@ -140,11 +149,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage += 1 def use_password_to_setup_ssh_key_pairs(self) -> None: - """Get the user password for the central server. - - If correct, SSH key pair is set up and 'OK' button changed - to 'Finish'. Otherwise, continue allowing failed password attempts. - """ + """ """ password = self.query_one("#setup_ssh_password_input").value success, output = self.interface.setup_key_pair_and_rclone_config( @@ -152,10 +157,25 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: ) if success: - message = "Connection successful! SSH key saved to the RClone config file." - self.query_one("#setup_ssh_ok_button").label = "Finish" - self.query_one("#setup_ssh_cancel_button").disabled = True - self.stage += 1 + if self.interface.project.cfg.connection_method_rclone_config_has_password(): + message = ( + "Password already set on config file, skipping password set up." + "To remove the password, call project.remove_rclone_password()" + "through the Python API." + ) + self.query_one("#setup_ssh_ok_button").label = "Yes" + self.query_one("#setup_ssh_cancel_button").visible = False + self.query_one("#setup_ssh_password_input").visible = False + self.stage += 2 # Go to final screen + else: + message = ( + "Would you like to use Windows Credential Manager to set a password on " + "the RClone config file on which your RClone is stored? ." + ) + self.query_one("#setup_ssh_ok_button").label = "Yes" + self.query_one("#setup_ssh_cancel_button").label = "No" + self.query_one("#setup_ssh_password_input").visible = False + self.stage += 1 # Go to password set up screen else: message = ( @@ -166,3 +186,25 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.failed_password_attempts += 1 self.query_one("#messagebox_message_label").update(message) + + def ask_setup_rclone_password(self): + """""" + success = self.interface.try_setup_rclone_password() + + if not self.interface.project.cfg.connection_method_rclone_config_has_password(): + self._try_set_rclone_password() + else: + pass + # show message + # message = "Connection successful! SSH key saved to the RClone config file." + + def show_connection_sucesssful_message(self): + """""" + self.query_one("#setup_ssh_ok_button").label = "Finish" + self.query_one("#setup_ssh_cancel_button").disabled = True + + message = ( + "Connection successful! SSH key saved to the RClone config file." + ) + self.query_one("#messagebox_message_label").update(message) + self.stage += 1 From b32a979ddd89da6e4f6756fc0931b583d1d307db Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sat, 4 Oct 2025 14:10:58 +0100 Subject: [PATCH 015/100] Adding password to SSH GUI. --- datashuttle/configs/config_class.py | 37 +++++++++++---- datashuttle/datashuttle_class.py | 51 +++++++++++--------- datashuttle/tui/interface.py | 2 +- datashuttle/tui/screens/setup_ssh.py | 70 ++++++++++++++++------------ datashuttle/utils/rclone.py | 4 +- 5 files changed, 99 insertions(+), 65 deletions(-) diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 3087e6ec6..f36669a72 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -67,24 +67,43 @@ def __init__( self.rclone_password_state_file_path = ( self.file_path.parent / "rclone_ps_state.yaml" ) - self.rclone_has_password = {} - self.setup_rclone_has_password() - def connection_method_rclone_config_has_password(self): - return self.rclone_has_password[self["connection_method"]] + def load_rclone_has_password(self): + assert self["connection_method"] in ["ssh", "aws", "gdrive"] - def setup_rclone_has_password(self): - """""" if self.rclone_password_state_file_path.is_file(): with open(self.rclone_password_state_file_path, "r") as file: - self.rclone_has_password = yaml.full_load(file) + rclone_has_password = yaml.full_load(file) else: - self.rclone_has_password = { + rclone_has_password = { "ssh": False, "gdrive": False, "aws": False, } - self.save_rclone_password_state() + + with open(self.rclone_password_state_file_path, "w") as file: + yaml.dump(rclone_has_password, file) + + return rclone_has_password + + def get_rclone_has_password( + self, + ): # TODO: hmm this is used a lot... could hold state.. but nice to save... + """""" + rclone_has_password = self.load_rclone_has_password() + + return rclone_has_password[self["connection_method"]] + + def set_rclone_has_password(self, value): + """""" + assert self["connection_method"] in ["ssh", "aws", "gdrive"] + + rclone_has_password = self.load_rclone_has_password() + + rclone_has_password[self["connection_method"]] = value + + with open(self.rclone_password_state_file_path, "w") as file: + yaml.dump(rclone_has_password, file) def setup_after_load(self) -> None: """Set up the config after loading it.""" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index a9e870678..1c04379ae 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -848,7 +848,7 @@ def setup_ssh_connection(self) -> None: f"{self.cfg.get_rclone_config_filepath()}.\n\n" ) - if not self.cfg.connection_method_rclone_config_has_password(): + if not self.cfg.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail( @@ -911,7 +911,7 @@ def setup_gdrive_connection(self) -> None: self.cfg, process, log=True ) - if not self.cfg.connection_method_rclone_config_has_password(): + if not self.cfg.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -944,7 +944,7 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) - if not self.cfg.connection_method_rclone_config_has_password(): + if not self.cfg.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -958,20 +958,27 @@ def setup_aws_connection(self) -> None: # Rclone config password # ------------------------------------------------------------------------- - def _try_set_rclone_password(self): + # TODO: LOAD AND SAVE CONFIG FILE ON EACH USE!! + + def _try_set_rclone_password( + self, ask_for_input=True + ): # TODO: handle this better """""" - pass_type = { - "Windows": "Windows credential manager", - "Linux": "the `pass` program", - "Darwin": "macOS inbuild `security`.", - } - - input_ = utils.get_user_input( - f"Would you like to set a password using {pass_type[platform.system()]}.\n" - f"Press 'y' to set password or leave blank to skip." - ) + if ask_for_input: + pass_type = { + "Windows": "Windows credential manager", + "Linux": "the `pass` program", + "Darwin": "macOS inbuild `security`.", + } + + input_ = utils.get_user_input( + f"Would you like to set a password using {pass_type[platform.system()]}.\n" + f"Press 'y' to set password or leave blank to skip." + ) - set_password = input_ == "y" + set_password = input_ == "y" + else: + set_password = True if set_password: try: @@ -988,9 +995,11 @@ def _try_set_rclone_password(self): utils.log_and_message("Password set successfully") + # TODO: REMOVE from (e) just print (e) + def set_rclone_password(self): """""" - if self.cfg.connection_method_rclone_config_has_password(): + if self.cfg.get_rclone_has_password(): raise RuntimeError( "This config file already has a password set. " "First, use `remove_rclone_password` to remove it." @@ -998,13 +1007,11 @@ def set_rclone_password(self): rclone_password.run_rclone_config_encrypt(self.cfg) - self.cfg.rclone_has_password[self.cfg["connection_method"]] = True - - self.cfg.save_rclone_password_state() + self.cfg.set_rclone_has_password(True) def remove_rclone_password(self): """""" - if not self.cfg.connection_method_rclone_config_has_password(): + if not self.cfg.get_rclone_has_password(): raise RuntimeError( f"The config for the current connection method: {self.cfg['connection_method']} " f"does not have a password. Cannot remove." @@ -1012,9 +1019,7 @@ def remove_rclone_password(self): rclone_password.remove_rclone_password(self.cfg) - self.cfg.rclone_has_password[self.cfg["connection_method"]] = False - - self.cfg.save_rclone_password_state() + self.cfg.set_rclone_has_password(False) # ------------------------------------------------------------------------- # Configs diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 6d50cebf7..b18d1c670 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -631,7 +631,7 @@ def setup_aws_connection( def try_setup_rclone_password(self): try: - self.project._try_set_password() + self.project._try_set_rclone_password(ask_for_input=False) return True, None except BaseException as e: return False, str(e) diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 7802e474e..31c723b3e 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -8,6 +8,8 @@ from datashuttle.tui.interface import Interface + +from textual import work from textual.containers import Container, Horizontal from textual.screen import ModalScreen from textual.widgets import ( @@ -29,7 +31,7 @@ def __init__(self, interface: Interface) -> None: super(SetupSshScreen, self).__init__() self.interface = interface - self.stage = 0 + self.stage = "init" self.failed_password_attempts = 1 self.key: paramiko.RSAKey @@ -67,28 +69,28 @@ def on_button_pressed(self, event: Button.pressed) -> None: input, multiple attempts are allowed. """ if event.button.id == "setup_ssh_cancel_button": - if self.stage == 3: - self.show_connection_sucesssful_message() + if self.stage == "show_success_message": + self.show_connection_successful_message() else: self.dismiss() if event.button.id == "setup_ssh_ok_button": - if self.stage == 0: + if self.stage == "init": self.ask_user_to_accept_hostkeys() - elif self.stage == 1: # TODO: use str for stages + elif self.stage == "save_hostkeys": self.save_hostkeys_and_prompt_password_input() - elif self.stage == 2: + elif self.stage == "ask_for_password": self.use_password_to_setup_ssh_key_pairs() - elif self.stage == 3: - self.ask_setup_rclone_password() + elif self.stage == "set_up_password": + self.try_setup_rclone_password() - elif self.stage == 4: - self.show_connection_sucesssful_message() + elif self.stage == "show_success_message": + self.show_connection_successful_message() - elif self.stage == 5: + elif self.stage == "finished": self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: @@ -121,7 +123,7 @@ def ask_user_to_accept_hostkeys(self) -> None: self.query_one("#setup_ssh_ok_button").disabled = True self.query_one("#messagebox_message_label").update(message) - self.stage += 1 + self.stage = "save_hostkeys" def save_hostkeys_and_prompt_password_input(self) -> None: """Get the user password for the central server. @@ -146,7 +148,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.query_one("#setup_ssh_ok_button").disabled = True self.query_one("#messagebox_message_label").update(message) - self.stage += 1 + self.stage = "ask_for_password" def use_password_to_setup_ssh_key_pairs(self) -> None: """ """ @@ -157,16 +159,16 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: ) if success: - if self.interface.project.cfg.connection_method_rclone_config_has_password(): + if self.interface.project.cfg.get_rclone_has_password(): message = ( "Password already set on config file, skipping password set up." "To remove the password, call project.remove_rclone_password()" "through the Python API." ) - self.query_one("#setup_ssh_ok_button").label = "Yes" - self.query_one("#setup_ssh_cancel_button").visible = False + self.query_one("#setup_ssh_ok_button").label = "Ok" + self.query_one("#setup_ssh_cancel_button").disabled = True self.query_one("#setup_ssh_password_input").visible = False - self.stage += 2 # Go to final screen + self.stage = "show_success_message" else: message = ( "Would you like to use Windows Credential Manager to set a password on " @@ -175,7 +177,7 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.query_one("#setup_ssh_ok_button").label = "Yes" self.query_one("#setup_ssh_cancel_button").label = "No" self.query_one("#setup_ssh_password_input").visible = False - self.stage += 1 # Go to password set up screen + self.stage = "set_up_password" # Go to password set up screen else: message = ( @@ -187,24 +189,32 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.query_one("#messagebox_message_label").update(message) - def ask_setup_rclone_password(self): + def try_setup_rclone_password(self): """""" - success = self.interface.try_setup_rclone_password() + success, output = self.interface.try_setup_rclone_password() - if not self.interface.project.cfg.connection_method_rclone_config_has_password(): - self._try_set_rclone_password() + if success: + message = "Password successfully set on the config file." + self.query_one("#messagebox_message_label").update(message) + self.query_one("#setup_ssh_ok_button").label = "Ok" + self.query_one( + "#setup_ssh_cancel_button" + ).label = "Cancel" # check this else: - pass - # show message - # message = "Connection successful! SSH key saved to the RClone config file." + message = f"The password set up failed. Exception: {output}" + self.query_one("#messagebox_message_label").update(message) - def show_connection_sucesssful_message(self): + self.stage = "show_success_message" + + @work(exclusive=True, thread=True) + def run_interface(self): + self.interface.try_setup_rclone_password() + + def show_connection_successful_message(self): """""" self.query_one("#setup_ssh_ok_button").label = "Finish" self.query_one("#setup_ssh_cancel_button").disabled = True - message = ( - "Connection successful! SSH key saved to the RClone config file." - ) + message = "Connection was set up successfully. SSH key saved to the RClone config file." self.query_one("#messagebox_message_label").update(message) - self.stage += 1 + self.stage = "finished" diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index c1addfa94..780dbbf4c 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -157,7 +157,7 @@ def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ - set_password = cfg.connection_method_rclone_config_has_password() + set_password = cfg.get_rclone_has_password() if set_password: rclone_password.set_credentials_as_password_command(cfg) @@ -562,7 +562,7 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.connection_method_rclone_config_has_password(): + if cfg.get_rclone_has_password(): print("REMOVED") rclone_password.remove_credentials_as_password_command() From d8364d3f54d283ba09df5ebd6c123137692e16f7 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 14:35:52 +0100 Subject: [PATCH 016/100] Updating setup ssh. --- datashuttle/tui/screens/setup_ssh.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 31c723b3e..e8e599932 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -161,8 +161,8 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: if success: if self.interface.project.cfg.get_rclone_has_password(): message = ( - "Password already set on config file, skipping password set up." - "To remove the password, call project.remove_rclone_password()" + "Password already set on config file, skipping password set up.\n\n" + "To remove the password, call `project.remove_rclone_password()` " "through the Python API." ) self.query_one("#setup_ssh_ok_button").label = "Ok" @@ -199,7 +199,8 @@ def try_setup_rclone_password(self): self.query_one("#setup_ssh_ok_button").label = "Ok" self.query_one( "#setup_ssh_cancel_button" - ).label = "Cancel" # check this + ).label = "Cancel" # check this# + self.query_one("#setup_ssh_cancel_button").disabled = True else: message = f"The password set up failed. Exception: {output}" self.query_one("#messagebox_message_label").update(message) From b9469d6678640427a05e711132473110fd22f7ab Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 14:36:08 +0100 Subject: [PATCH 017/100] Playing with setup gdrive. --- datashuttle/tui/screens/setup_gdrive.py | 40 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 509ac1a52..d0e7cd783 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -44,7 +44,9 @@ def __init__(self, interface: Interface) -> None: id="setup_gdrive_generic_input_box", placeholder="Enter value here", ) - self.enter_button = Button("Enter", id="setup_gdrive_enter_button") + self.enter_button = Button( + "Enter", id="setup_gdrive_no_browser_enter_button" + ) def compose(self) -> ComposeResult: """Add widgets to the SetupGdriveScreen.""" @@ -102,18 +104,18 @@ def on_button_pressed(self, event: Button.Pressed) -> None: else: self.ask_user_for_browser() - elif event.button.id == "setup_gdrive_yes_button": - self.remove_yes_no_buttons() + elif event.button.id == "setup_gdrive_has_browser_yes_button": + self.remove_yes_no_browser_buttons() self.open_browser_and_setup_gdrive_connection( self.gdrive_client_secret ) elif event.button.id == "setup_gdrive_no_button": self.is_browser_available = False - self.remove_yes_no_buttons() + self.remove_yes_no_browser_buttons() self.prompt_user_for_config_token() - elif event.button.id == "setup_gdrive_enter_button": + elif event.button.id == "setup_gdrive_no_browser_enter_button": if ( self.interface.project.cfg["gdrive_client_id"] and self.stage == 0 @@ -150,7 +152,7 @@ def ask_user_for_browser(self) -> None: self.input_box.visible = False # Mount the Yes and No buttons - yes_button = Button("Yes", id="setup_gdrive_yes_button") + yes_button = Button("Yes", id="setup_gdrive_has_browser_yes_button") no_button = Button("No", id="setup_gdrive_no_button") self.query_one("#setup_gdrive_buttons_horizontal").mount( @@ -211,7 +213,9 @@ def prompt_user_for_config_token(self) -> None: message + "\nPress shift+click to copy." ) - self.enter_button = Button("Enter", id="setup_gdrive_enter_button") + self.enter_button = Button( + "Enter", id="setup_gdrive_no_browser_enter_button" + ) self.query_one("#setup_gdrive_buttons_horizontal").mount( self.enter_button, before="#setup_gdrive_cancel_button" ) @@ -264,7 +268,8 @@ async def setup_gdrive_connection_and_update_ui( success, output = worker.result if success: - self.show_finish_screen() + # self.show_finish_screen() + self.show_password_screen() else: self.input_box.disabled = False self.enter_button.disabled = False @@ -302,6 +307,21 @@ def show_finish_screen(self) -> None: Button("Finish", id="setup_gdrive_finish_button") ) + def show_password_screen(self): + """""" + assert False + self.show_password_screen() + self.remove_yes_no_browser_buttons() + self.query_one("setup_gdrive_cancel_button").remove() + yes_button = Button("Yes", id="setup_gdrive_set_password_yes_button") + no_button = Button("No", id="setup_gdrive_set_password_no_button") + + self.query_one("#setup_gdrive_buttons_horizontal").mount( + yes_button, no_button + ) + message = "Would you like to set a password?" + self.update_message_box_message(message) + def display_failed(self, output) -> None: """Update the message box indicating the set up failed.""" message = ( @@ -329,7 +349,7 @@ def mount_input_box_before_buttons( self.input_box.visible = True self.input_box.value = "" - def remove_yes_no_buttons(self) -> None: + def remove_yes_no_browser_buttons(self) -> None: """Remove yes and no buttons.""" - self.query_one("#setup_gdrive_yes_button").remove() + self.query_one("#setup_gdrive_has_browser_yes_button").remove() self.query_one("#setup_gdrive_no_button").remove() From e2d2bf7c2c5acd384a63cbe43af58a3e9424ddf2 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 15:22:40 +0100 Subject: [PATCH 018/100] Setup gdrive 2. --- datashuttle/tui/screens/setup_gdrive.py | 48 +++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index d0e7cd783..c1fb0d901 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -89,6 +89,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: if ( event.button.id == "setup_gdrive_cancel_button" or event.button.id == "setup_gdrive_finish_button" + or event.button.id == "setup_gdrive_set_password_no_button" ): # see setup_gdrive_connection_and_update_ui() if self.setup_worker and self.setup_worker.is_running: @@ -137,6 +138,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.gdrive_client_secret, config_token ) + # elif event.button.id == "setup_gdrive_set_password_yes_button": + # self.set_password() + + def ask_user_for_browser(self) -> None: """Ask the user if their machine has access to a browser.""" message = ( @@ -225,7 +230,7 @@ def setup_gdrive_connection_using_config_token( self, gdrive_client_secret: str | None, config_token: str | None ) -> None: """Set up the Google Drive connection using rclone config token.""" - message = "Setting up connection." + message = "Setting up connection..." self.update_message_box_message(message) asyncio.create_task( @@ -253,7 +258,7 @@ async def setup_gdrive_connection_and_update_ui( The rclone process object is stored in the `Interface` class to handle closing the process as the thread does not kill the process itself upon cancellation and the process is awaited ensure that the process finishes and any raised errors are caught. - Therefore, the worker thread thread and the rclone process are separately cancelled + Therefore, the worker thread and the rclone process are separately cancelled when the user presses the cancel button. (see `on_button_pressed`) """ self.input_box.disabled = True @@ -268,8 +273,8 @@ async def setup_gdrive_connection_and_update_ui( success, output = worker.result if success: - # self.show_finish_screen() - self.show_password_screen() + pass + # self.show_password_screen() else: self.input_box.disabled = False self.enter_button.disabled = False @@ -297,7 +302,7 @@ def setup_gdrive_connection( # UI Update Methods # ---------------------------------------------------------------------------------- - def show_finish_screen(self) -> None: + def show_finish_screen(self) -> None: # TODO: NOW DUPLCIATE """Show the final screen after successful set up.""" message = "Setup Complete!" self.query_one("#setup_gdrive_cancel_button").remove() @@ -309,18 +314,39 @@ def show_finish_screen(self) -> None: def show_password_screen(self): """""" - assert False - self.show_password_screen() - self.remove_yes_no_browser_buttons() - self.query_one("setup_gdrive_cancel_button").remove() + # self.remove_yes_no_browser_buttons() + self.query_one("#setup_gdrive_cancel_button").remove() + + message = "Would you like to set a password?" + self.update_message_box_message(message) + yes_button = Button("Yes", id="setup_gdrive_set_password_yes_button") no_button = Button("No", id="setup_gdrive_set_password_no_button") self.query_one("#setup_gdrive_buttons_horizontal").mount( yes_button, no_button ) - message = "Would you like to set a password?" - self.update_message_box_message(message) + + # TODO: DIRECT COPY + def set_password(self): + """""" + success, output = self.interface.try_setup_rclone_password() + + if success: + message = "Password successfully set on the config file. Setup complete!" + message = "Setup Complete!" + self.query_one("#setup_gdrive_cancel_button").remove() + + self.update_message_box_message(message) + self.query_one("#setup_gdrive_buttons_horizontal").mount( + Button("Finish", id="setup_gdrive_finish_button") + ) + else: + message = f"The password set up failed. Exception: {output}" + self.query_one("#messagebox_message_label").update(message) + + self.stage = "show_success_message" + def display_failed(self, output) -> None: """Update the message box indicating the set up failed.""" From 6ad0d6b8213c94fc36fe0c42b023d72cc5f61d2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Oct 2025 14:23:01 +0000 Subject: [PATCH 019/100] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datashuttle/tui/screens/setup_gdrive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index c1fb0d901..b900f21fd 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -139,8 +139,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) # elif event.button.id == "setup_gdrive_set_password_yes_button": - # self.set_password() - + # self.set_password() def ask_user_for_browser(self) -> None: """Ask the user if their machine has access to a browser.""" @@ -333,7 +332,9 @@ def set_password(self): success, output = self.interface.try_setup_rclone_password() if success: - message = "Password successfully set on the config file. Setup complete!" + message = ( + "Password successfully set on the config file. Setup complete!" + ) message = "Setup Complete!" self.query_one("#setup_gdrive_cancel_button").remove() @@ -347,7 +348,6 @@ def set_password(self): self.stage = "show_success_message" - def display_failed(self, output) -> None: """Update the message box indicating the set up failed.""" message = ( From c2b6e19775fb4ffb8822ffbb868fa0c807054f22 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 20:06:22 +0100 Subject: [PATCH 020/100] Rough version working on all screens. --- datashuttle/datashuttle_class.py | 5 ++- datashuttle/tui/interface.py | 6 ++- datashuttle/tui/screens/setup_aws.py | 55 ++++++++++++++++++------- datashuttle/tui/screens/setup_gdrive.py | 51 ++++++++++++++--------- datashuttle/tui/screens/setup_ssh.py | 27 ++++-------- datashuttle/utils/rclone.py | 38 ++++++++++++++--- datashuttle/utils/rclone_password.py | 10 +++++ datashuttle/utils/utils.py | 15 ++----- 8 files changed, 136 insertions(+), 71 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 1c04379ae..bc419d951 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -907,6 +907,9 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) + print("got process") + + # TODO: do something with stderr stdout here, in general handle errors better! rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( self.cfg, process, log=True ) @@ -987,10 +990,10 @@ def _try_set_rclone_password( config_path = self.cfg.get_rclone_config_filepath() utils.log_and_raise_error( + f"{str(e)}\n" f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" f"Use set_rclone_password()` to attempt to set the password again (see full error message above). ", RuntimeError, - from_error=e, ) utils.log_and_message("Password set successfully") diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index b18d1c670..5e5bedbf2 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -597,7 +597,11 @@ def await_successful_gdrive_connection_setup_raise_on_fail( The `self.gdrive_setup_process_killed` flag helps prevent raising errors in case the process was killed manually. """ - stdout, stderr = process.communicate() + stdout, stderr = ( + rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( + self.project.cfg, process, log=False + ) + ) if not self.gdrive_setup_process_killed: if process.returncode != 0: diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 8527e1dd9..c1aec7a0f 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -26,7 +26,7 @@ def __init__(self, interface: Interface) -> None: super(SetupAwsScreen, self).__init__() self.interface = interface - self.stage = 0 + self.stage = "init" def compose(self) -> ComposeResult: """Set widgets on the SetupAwsScreen.""" @@ -54,16 +54,26 @@ def on_mount(self) -> None: def on_button_pressed(self, event: Button.Pressed) -> None: """Handle button press on the screen.""" if event.button.id == "setup_aws_cancel_button": - self.dismiss() + if self.stage == "ask_password": + message = "AWS Connection Successful!" # TODO: MOVE ADD + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_ok_button").label = "Finish" + self.query_one("#setup_aws_cancel_button").remove() + self.stage = "finished" + else: + self.dismiss() if event.button.id == "setup_aws_ok_button": - if self.stage == 0: + if self.stage == "init": self.prompt_user_for_aws_secret_access_key() - elif self.stage == 1: + elif self.stage == "use_secret_access_key": self.use_secret_access_key_to_setup_aws_connection() - elif self.stage == 2: + elif self.stage == "ask_password": + self.ask_for_password() + + elif self.stage == "finished": self.dismiss() def prompt_user_for_aws_secret_access_key(self) -> None: @@ -73,7 +83,7 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").visible = True - self.stage += 1 + self.stage = "use_secret_access_key" def use_secret_access_key_to_setup_aws_connection(self) -> None: """Set up the AWS connection and inform user of success or failure.""" @@ -86,21 +96,36 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: ) if success: - message = "AWS Connection Successful!" - self.query_one( - "#setup_aws_secret_access_key_input" - ).visible = False + message = "Would you like to set a password?" + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_secret_access_key_input").remove() + self.query_one("#setup_aws_ok_button").label = "Yes" + self.query_one("#setup_aws_cancel_button").label = "No" + self.stage = "ask_password" else: message = ( - f"AWS setup failed. Please check your configs and secret access key" + f"AWS setup failed. Please check your configs and secret access key" # TODO: check this f"\n\n Traceback: {output}" ) self.query_one( "#setup_aws_secret_access_key_input" ).disabled = True - self.query_one("#setup_aws_ok_button").label = "Finish" - self.query_one("#setup_aws_messagebox_message").update(message) - self.query_one("#setup_aws_cancel_button").disabled = True - self.stage += 1 + self.query_one("#setup_aws_ok_button").label = "Retry" + self.query_one("#setup_aws_messagebox_message").update(message) + + # TODO: this is a direct copy + def ask_for_password(self): # TODO: CHANGE NAME + """""" + success, output = self.interface.try_setup_rclone_password() + + if success: + message = "The password was successfully set. Setup complete!" + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_ok_button").label = "Finish" + self.query_one("#setup_aws_cancel_button").disabled = True + self.stage = "finished" + else: + message = f"The password set up failed. Exception: {output}" + self.query_one("#setup_aws_messagebox_message").update(message) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index b900f21fd..45969dc2f 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -89,7 +89,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: if ( event.button.id == "setup_gdrive_cancel_button" or event.button.id == "setup_gdrive_finish_button" - or event.button.id == "setup_gdrive_set_password_no_button" ): # see setup_gdrive_connection_and_update_ui() if self.setup_worker and self.setup_worker.is_running: @@ -138,8 +137,17 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.gdrive_client_secret, config_token ) - # elif event.button.id == "setup_gdrive_set_password_yes_button": - # self.set_password() + elif event.button.id == "setup_gdrive_set_password_yes_button": + self.set_password() + + elif event.button.id == "setup_gdrive_skip_password_ok_button": + self.query_one("#setup_gdrive_skip_password_ok_button").remove() + self.set_finish_page() + + elif event.button.id == "setup_gdrive_set_password_no_button": + self.query_one("#setup_gdrive_set_password_yes_button").remove() + self.query_one("#setup_gdrive_set_password_no_button").remove() + self.set_finish_page() def ask_user_for_browser(self) -> None: """Ask the user if their machine has access to a browser.""" @@ -194,12 +202,16 @@ def open_browser_and_setup_gdrive_connection( message = "Please authenticate through browser (it should open automatically)." self.update_message_box_message(message) - asyncio.create_task( + task = asyncio.create_task( self.setup_gdrive_connection_and_update_ui( gdrive_client_secret=gdrive_client_secret ), name="setup_gdrive_connection_with_browser_task", ) + task.add_done_callback(self.handle_task_result) + + def handle_task_result(self, task): + task.result() def prompt_user_for_config_token(self) -> None: """Prompt the user for the rclone config token for Google Drive setup.""" @@ -272,8 +284,17 @@ async def setup_gdrive_connection_and_update_ui( success, output = worker.result if success: - pass - # self.show_password_screen() + self.show_password_screen() + for id in [ + "#setup_gdrive_cancel_button", + "#setup_gdrive_generic_input_box", + "#setup_gdrive_no_browser_enter_button", + ]: + try: + widget = self.query_one(id) + await widget.remove() + except BaseException: + pass # TODO else: self.input_box.disabled = False self.enter_button.disabled = False @@ -301,10 +322,9 @@ def setup_gdrive_connection( # UI Update Methods # ---------------------------------------------------------------------------------- - def show_finish_screen(self) -> None: # TODO: NOW DUPLCIATE + def set_finish_page(self) -> None: # TODO: NOW DUPLCIATE """Show the final screen after successful set up.""" message = "Setup Complete!" - self.query_one("#setup_gdrive_cancel_button").remove() self.update_message_box_message(message) self.query_one("#setup_gdrive_buttons_horizontal").mount( @@ -313,9 +333,6 @@ def show_finish_screen(self) -> None: # TODO: NOW DUPLCIATE def show_password_screen(self): """""" - # self.remove_yes_no_browser_buttons() - self.query_one("#setup_gdrive_cancel_button").remove() - message = "Would you like to set a password?" self.update_message_box_message(message) @@ -332,21 +349,17 @@ def set_password(self): success, output = self.interface.try_setup_rclone_password() if success: - message = ( - "Password successfully set on the config file. Setup complete!" - ) - message = "Setup Complete!" - self.query_one("#setup_gdrive_cancel_button").remove() + self.query_one("#setup_gdrive_set_password_yes_button").remove() + self.query_one("#setup_gdrive_set_password_no_button").remove() + message = "The password was successfully set. Setup complete!" self.update_message_box_message(message) self.query_one("#setup_gdrive_buttons_horizontal").mount( Button("Finish", id="setup_gdrive_finish_button") ) else: message = f"The password set up failed. Exception: {output}" - self.query_one("#messagebox_message_label").update(message) - - self.stage = "show_success_message" + self.update_message_box_message(message) def display_failed(self, output) -> None: """Update the message box indicating the set up failed.""" diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index e8e599932..68532b58c 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -159,25 +159,14 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: ) if success: - if self.interface.project.cfg.get_rclone_has_password(): - message = ( - "Password already set on config file, skipping password set up.\n\n" - "To remove the password, call `project.remove_rclone_password()` " - "through the Python API." - ) - self.query_one("#setup_ssh_ok_button").label = "Ok" - self.query_one("#setup_ssh_cancel_button").disabled = True - self.query_one("#setup_ssh_password_input").visible = False - self.stage = "show_success_message" - else: - message = ( - "Would you like to use Windows Credential Manager to set a password on " - "the RClone config file on which your RClone is stored? ." - ) - self.query_one("#setup_ssh_ok_button").label = "Yes" - self.query_one("#setup_ssh_cancel_button").label = "No" - self.query_one("#setup_ssh_password_input").visible = False - self.stage = "set_up_password" # Go to password set up screen + message = ( + "Would you like to use Windows Credential Manager to set a password on " + "the RClone config file on which your RClone is stored? ." + ) + self.query_one("#setup_ssh_ok_button").label = "Yes" + self.query_one("#setup_ssh_cancel_button").label = "No" + self.query_one("#setup_ssh_password_input").visible = False + self.stage = "set_up_password" # Go to password set up screen else: message = ( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 780dbbf4c..e227133d1 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -122,18 +122,24 @@ def call_rclone_with_popen_for_central_connection( It is not possible to kill a process while running it using `subprocess.run`. Killing a process might be required when running rclone setup in a thread worker to allow the user to cancel the setup process. In such a case, cancelling the - thread worker alone will not kill the rclone process, so we need to kill the + thread worker alone will not kill the rclone process, so we need to kill thenothe env + process explicitly. """ + # if cfg.get_rclone_has_password(): + # rclone_password.set_credentials_as_password_command(cfg) + command = "rclone " + command process = subprocess.Popen( - shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE + shlex.split(command), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, # , env=copy.deepcopy(os.environ) ) return process def await_call_rclone_with_popen_for_central_connection_raise_on_fail( - process: subprocess.Popen, log: bool = True + cfg: Configs, process: subprocess.Popen, log: bool = True ): """Await rclone the subprocess.Popen call. @@ -152,21 +158,27 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( if log: log_rclone_config_output() + return stdout, stderr + def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ set_password = cfg.get_rclone_has_password() + print("set_password", set_password) if set_password: rclone_password.set_credentials_as_password_command(cfg) + print("os env", os.environ) + results = lambda_func() if set_password: rclone_password.remove_credentials_as_password_command() + print("res") return results @@ -244,6 +256,7 @@ def setup_rclone_config_for_ssh( ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() + cfg.set_rclone_has_password(False) command = ( f"config create " @@ -340,6 +353,14 @@ def setup_rclone_config_for_gdrive( else "" ) + rclone_config_filepath = get_full_config_filepath( + cfg + ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.set_rclone_has_password(False) + + print("Trying to create gdrive config") process = call_rclone_with_popen_for_central_connection( f"config create " f"{rclone_config_name} " @@ -349,9 +370,9 @@ def setup_rclone_config_for_gdrive( f"scope drive " f"root_folder_id {cfg['gdrive_root_folder_id']} " f"{extra_args} " - f"{get_config_arg(cfg)}" + f"{get_config_arg(cfg)}", ) - + print("Created gdrive config") return process @@ -390,6 +411,13 @@ def setup_rclone_config_for_aws( else f" location_constraint {aws_region}" ) + rclone_config_filepath = get_full_config_filepath( + cfg + ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.set_rclone_has_password(False) + output = call_rclone( "config create " f"{rclone_config_name} " diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index fabebfb40..2c92d3dcb 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -43,6 +43,8 @@ def set_password_windows(cfg: Configs): "(ConvertTo-SecureString ([System.Web.Security.Membership]::GeneratePassword(40,10)) -AsPlainText -Force) " f"| Export-Clixml -LiteralPath '{password_filepath}'" ) + print("set password cmd windows: ", ps_cmd) + output = subprocess.run( [shell, "-NoProfile", "-Command", ps_cmd], capture_output=True, @@ -125,6 +127,8 @@ def set_credentials_as_password_command(cfg): if platform.system() == "Windows": password_filepath = get_password_filepath(cfg) + print("password_filepath ", password_filepath) + assert password_filepath.exists(), ( "Critical error: password file not found when setting password command." ) @@ -142,6 +146,9 @@ def set_credentials_as_password_command(cfg): f"[System.Runtime.InteropServices.Marshal]::SecureStringToBSTR(" f"(Import-Clixml -LiteralPath '{password_filepath}' ).Password)))\"" ) + + print("setting rclone cmd: ", cmd) + os.environ["RCLONE_PASSWORD_COMMAND"] = cmd elif platform.system() == "Linux": @@ -208,6 +215,9 @@ def remove_rclone_password(cfg): remove_credentials_as_password_command() + if platform.system() == "Windows": + get_password_filepath(cfg).unlink() + utils.log_and_message( f"Password removed from rclone config file: {config_filepath}" ) diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 967a86038..c6047a3ae 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -45,15 +45,13 @@ def log_and_message(message: str, use_rich: bool = False) -> None: print_message_to_user(message, use_rich) -def log_and_raise_error( - message: str, exception: Any, from_error: Exception | None = None -) -> None: +def log_and_raise_error(message: str, exception: Any) -> None: """Log the message before raising the same message as an error.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.error(f"\n\n{' '.join(traceback.format_stack(limit=5))}") logger.error(message) - raise_error(message, exception, from_error=from_error) + raise_error(message, exception) def warn(message: str, log: bool) -> None: @@ -74,9 +72,7 @@ def warn(message: str, log: bool) -> None: warnings.warn(message) -def raise_error( - message: str, exception, from_error: Exception | None = None -) -> None: +def raise_error(message: str, exception) -> None: """Centralized way to raise an error. The logger is closed to ensure it is not still running @@ -84,10 +80,7 @@ def raise_error( """ ds_logger.close_log_filehandler() - if from_error: - raise exception(message) from from_error - else: - raise exception(message) + raise exception(message) def print_message_to_user( From 6a12c54d8ac5352705a25333e0e870ed175781bb Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 21:07:33 +0100 Subject: [PATCH 021/100] Rough version working on all screens2. --- datashuttle/tui/screens/setup_aws.py | 23 ++++++++++++++++++----- datashuttle/tui/screens/setup_ssh.py | 12 ++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index c1aec7a0f..88a1abb31 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -52,10 +52,17 @@ def on_mount(self) -> None: self.query_one("#setup_aws_secret_access_key_input").visible = False def on_button_pressed(self, event: Button.Pressed) -> None: - """Handle button press on the screen.""" + """Handle button press on the screen. + + The `setup_aws_ok_button` is used for all 'positive' events ('Yes, Ok') + and 'setup_aws_cancel_button' is used for 'negative' events ('No', 'Cancel'). + The appropriate action to take on the button press is determined by the + current stage. + + """ if event.button.id == "setup_aws_cancel_button": if self.stage == "ask_password": - message = "AWS Connection Successful!" # TODO: MOVE ADD + message = "AWS Connection Successful!" # self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" self.query_one("#setup_aws_cancel_button").remove() @@ -63,7 +70,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: else: self.dismiss() - if event.button.id == "setup_aws_ok_button": + elif event.button.id == "setup_aws_ok_button": + if self.stage == "init": self.prompt_user_for_aws_secret_access_key() @@ -83,10 +91,15 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").visible = True + self.query_one("#setup_aws_ok_button") + self.stage = "use_secret_access_key" def use_secret_access_key_to_setup_aws_connection(self) -> None: - """Set up the AWS connection and inform user of success or failure.""" + """Set up the AWS connection and failure. If success, move onto the + password screen. + + """ secret_access_key = self.query_one( "#setup_aws_secret_access_key_input" ).value @@ -124,7 +137,7 @@ def ask_for_password(self): # TODO: CHANGE NAME message = "The password was successfully set. Setup complete!" self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" - self.query_one("#setup_aws_cancel_button").disabled = True + self.query_one("#setup_aws_cancel_button").remove() self.stage = "finished" else: message = f"The password set up failed. Exception: {output}" diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 68532b58c..fbe72cf1f 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -69,7 +69,7 @@ def on_button_pressed(self, event: Button.pressed) -> None: input, multiple attempts are allowed. """ if event.button.id == "setup_ssh_cancel_button": - if self.stage == "show_success_message": + if self.stage == "set_up_password": self.show_connection_successful_message() else: self.dismiss() @@ -186,10 +186,7 @@ def try_setup_rclone_password(self): message = "Password successfully set on the config file." self.query_one("#messagebox_message_label").update(message) self.query_one("#setup_ssh_ok_button").label = "Ok" - self.query_one( - "#setup_ssh_cancel_button" - ).label = "Cancel" # check this# - self.query_one("#setup_ssh_cancel_button").disabled = True + self.query_one("#setup_ssh_cancel_button").remove() else: message = f"The password set up failed. Exception: {output}" self.query_one("#messagebox_message_label").update(message) @@ -203,7 +200,10 @@ def run_interface(self): def show_connection_successful_message(self): """""" self.query_one("#setup_ssh_ok_button").label = "Finish" - self.query_one("#setup_ssh_cancel_button").disabled = True + try: + self.query_one("#setup_ssh_cancel_button").remove() + except BaseException: + pass message = "Connection was set up successfully. SSH key saved to the RClone config file." self.query_one("#messagebox_message_label").update(message) From 8f42d67ac159e7a5d99f124b6b2c45ccde461f58 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 6 Oct 2025 21:25:49 +0100 Subject: [PATCH 022/100] Start tidying up TUI. --- datashuttle/tui/screens/setup_ssh.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index fbe72cf1f..163c226d0 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -23,7 +23,12 @@ class SetupSshScreen(ModalScreen): """Dialog window that sets up an SSH connection. This asks to confirm the central hostkey, and takes password to setup - SSH key pair. Under the hood uses `project.setup_ssh_connection()`. + SSH key pair as well as setting a password to the RClone config. + + Due to how textual works, it is simples for each button press to + trigger an action (e.g. set up host key) and then set up the widgets + for the next screen. Then, when the next button is pressed, we can + continue in this way of managing the screens. """ def __init__(self, interface: Interface) -> None: @@ -151,7 +156,13 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage = "ask_for_password" def use_password_to_setup_ssh_key_pairs(self) -> None: - """ """ + """Set up the SSH key pair using the user-supplied password + to the central server. + + Next, set up the request asking if they would like to set + a (separate) password on their RClone config, using the + system credential manager. + """ password = self.query_one("#setup_ssh_password_input").value success, output = self.interface.setup_key_pair_and_rclone_config( @@ -160,6 +171,7 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: if success: message = ( + "Connection set up successfully.\n" "Would you like to use Windows Credential Manager to set a password on " "the RClone config file on which your RClone is stored? ." ) @@ -179,7 +191,9 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.query_one("#messagebox_message_label").update(message) def try_setup_rclone_password(self): - """""" + """Try and set up a password to the RClone config using the system + credential manager. If successful, the next screen confirms success. + """ success, output = self.interface.try_setup_rclone_password() if success: From b7348baab573edca19ab190ebde04e9e2c4678ca Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 7 Oct 2025 16:07:19 +0100 Subject: [PATCH 023/100] Refactoring Rclone Configs. --- datashuttle/configs/canonical_folders.py | 21 ++- datashuttle/configs/config_class.py | 97 +------------ datashuttle/configs/rclone_configs.py | 134 ++++++++++++++++++ datashuttle/datashuttle_class.py | 20 +-- datashuttle/tui/interface.py | 2 +- datashuttle/utils/aws.py | 2 +- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/folders.py | 2 +- datashuttle/utils/rclone.py | 30 ++-- datashuttle/utils/rclone_password.py | 14 +- .../pages/get_started/set-up-a-project.md | 31 ++++ 11 files changed, 222 insertions(+), 133 deletions(-) create mode 100644 datashuttle/configs/rclone_configs.py diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index 52dee4de1..b2cb180a6 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -96,10 +96,19 @@ def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: def get_rclone_config_base_path(): - """TODO PLACEHOLDER.""" + """Get the path to the Rclone config file. This is used for + RClone config files for transfer targets (ssh, aws, gdrive). + This should match where RClone itself stores the config by default, + as described here: https://rclone.org/docs/#config-string + + Because RClone's resolution is a little complex, in some rare cases the + below may not match where RClone stores its configs. This just means that + local filesystem configs and transfer configs are stored in a separate place, + which is not a huge deal. + """ if platform.system() == "Windows": - return ( - Path().home() / "AppData" / "Roaming" / "rclone" - ) # # "$HOME/.config/rclone/rclone.conf") - else: # TODO HANDLE platform.system() == "Linux": - return Path().home() / ".config" / "rclone" + appdata_path = Path().home() / "AppData" / "Roaming" + if appdata_path.is_dir(): + return appdata_path / "rclone" + + return Path().home() / ".config" / "rclone" diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index f36669a72..638fac9f1 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -1,12 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Optional, Union, cast +from typing import TYPE_CHECKING, Dict, Union, cast if TYPE_CHECKING: from collections.abc import ItemsView, KeysView, ValuesView from datashuttle.utils.custom_types import ( - OverwriteExistingFiles, TopLevelFolder, ) @@ -20,6 +19,7 @@ canonical_configs, canonical_folders, load_configs, + rclone_configs, ) from datashuttle.utils import folders, utils @@ -37,6 +37,9 @@ def __init__( ) -> None: """Initialize the Configs class with project name, file path, and config dictionary. + This class also holds `RCloneConfigs` under the `.rclone` attribute to manage + the configs + Parameters ---------- project_name @@ -64,46 +67,7 @@ def __init__( self.hostkeys_path: Path self.project_metadata_path: Path - self.rclone_password_state_file_path = ( - self.file_path.parent / "rclone_ps_state.yaml" - ) - - def load_rclone_has_password(self): - assert self["connection_method"] in ["ssh", "aws", "gdrive"] - - if self.rclone_password_state_file_path.is_file(): - with open(self.rclone_password_state_file_path, "r") as file: - rclone_has_password = yaml.full_load(file) - else: - rclone_has_password = { - "ssh": False, - "gdrive": False, - "aws": False, - } - - with open(self.rclone_password_state_file_path, "w") as file: - yaml.dump(rclone_has_password, file) - - return rclone_has_password - - def get_rclone_has_password( - self, - ): # TODO: hmm this is used a lot... could hold state.. but nice to save... - """""" - rclone_has_password = self.load_rclone_has_password() - - return rclone_has_password[self["connection_method"]] - - def set_rclone_has_password(self, value): - """""" - assert self["connection_method"] in ["ssh", "aws", "gdrive"] - - rclone_has_password = self.load_rclone_has_password() - - rclone_has_password[self["connection_method"]] = value - - with open(self.rclone_password_state_file_path, "w") as file: - yaml.dump(rclone_has_password, file) + self.rclone = rclone_configs.RCloneConfigs(self, self.file_path.parent) def setup_after_load(self) -> None: """Set up the config after loading it.""" @@ -294,55 +258,6 @@ def get_base_folder( return base_folder - def get_rclone_config_name( - self, connection_method: Optional[str] = None - ) -> str: - """Generate the rclone configuration name for the central project. - - These configs are created by datashuttle but managed and stored by rclone. - """ - if connection_method is None: - connection_method = self["connection_method"] - - assert connection_method != "local_only", ( - "This state assumes a central connection." - ) - - return f"central_{self.project_name}_{connection_method}" - - def get_rclone_config_filepath(self) -> Path: - """""" - return ( - canonical_folders.get_rclone_config_base_path() - / f"{self.get_rclone_config_name()}.conf" - ) - - def make_rclone_transfer_options( - self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool - ) -> Dict: - """Create a dictionary of rclone transfer options. - - Originally these arguments were collected from configs, but now - they are passed via function arguments. The `show_transfer_progress` - and `dry_run` options are fixed here. - """ - allowed_overwrite = ["never", "always", "if_source_newer"] - - if overwrite_existing_files not in allowed_overwrite: - utils.log_and_raise_error( - f"`overwrite_existing_files` not " - f"recognised, must be one of: " - f"{allowed_overwrite}", - ValueError, - ) - - return { - "overwrite_existing_files": overwrite_existing_files, - "show_transfer_progress": True, - "transfer_verbosity": "vv", - "dry_run": dry_run, - } - def init_paths(self) -> None: """Initialize paths used by datashuttle.""" self.project_metadata_path = self["local_path"] / ".datashuttle" diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py new file mode 100644 index 000000000..e2b1a79ef --- /dev/null +++ b/datashuttle/configs/rclone_configs.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Optional + +if TYPE_CHECKING: + from datashuttle.utils.custom_types import ( + OverwriteExistingFiles, + ) + +from pathlib import Path + +import yaml + +from datashuttle.configs import canonical_folders +from datashuttle.utils import utils + + +class RCloneConfigs: + """This class manages the RClone configuration file. This is a file that RClone creates + to hold all information about local and remote transfer targets. For example, the + ssh RClone config holds the private key. + + In datashuttle, local filesystem configs uses the Rclone default configuration file, + that RClone manages. However, remote transfers to ssh, aws and gdrive are held in + separate config files (set using RClone's --config argument). Then being separate + means passwords can be set on these files. + + This class tracks the state on whether a RClone config has a password, as well + as provides the default names for the rclone conf (e.g. central__). + + Parameters + ---------- + config_base_class + Path to the datashuttle configs folder where all configs for the project are stored. + + """ + + def __init__(self, datashuttle_configs, config_base_path): + self.datashuttle_configs = configs + self.rclone_password_state_file_path = ( + config_base_path / "rclone_ps_state.yaml" + ) + + def load_rclone_has_password(self): + """Track whether the Rclone config file has a password set. This could be + read directly from the RClone config file, but requires a subprocess call + which can be slow on Windows. As this function is called a lot, we track + this explicitly when a rclone config password is set / removed + and store to disk between sessions. + """ + assert self.datashuttle_configs["connection_method"] in [ + "ssh", + "aws", + "gdrive", + ] + + if self.rclone_password_state_file_path.is_file(): + with open(self.rclone_password_state_file_path, "r") as file: + rclone_has_password = yaml.full_load(file) + else: + rclone_has_password = { + "ssh": False, + "gdrive": False, + "aws": False, + } + + with open(self.rclone_password_state_file_path, "w") as file: + yaml.dump(rclone_has_password, file) + + return rclone_has_password + + def set_rclone_has_password(self, value): + """Store the current state of the rclone config file password for the `connection_method`. + + Note that this is stored to disk each call (rather than tracked locally) to ensure + it is updated live if updated through the Python API while the TUI is also running. + """ + assert self["connection_method"] in ["ssh", "aws", "gdrive"] + + rclone_has_password = self.load_rclone_has_password() + + rclone_has_password[self.datashuttle_configs["connection_method"]] = ( + value + ) + + with open(self.rclone_password_state_file_path, "w") as file: + yaml.dump(rclone_has_password, file) + + def get_rclone_has_password( + self, + ): + """Return whether the config file associated with the current `connection_method`.""" + rclone_has_password = self.load_rclone_has_password() + + return rclone_has_password[ + self.datashuttle_configs["connection_method"] + ] + + def get_rclone_config_name( + self, connection_method: Optional[str] = None + ) -> str: + """Generate the rclone configuration name for the central project.""" + if connection_method is None: + connection_method = self.datashuttle_configs["connection_method"] + + return f"central_{self.project_name}_{connection_method}" + + def get_rclone_config_filepath(self) -> Path: + """The full filepath to the rclone `.conf` config file""" + return ( + canonical_folders.get_rclone_config_base_path() + / f"{self.get_rclone_config_name()}.conf" + ) + + def make_rclone_transfer_options( + self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool + ) -> Dict: + """Create a dictionary of rclone transfer options.""" + allowed_overwrite = ["never", "always", "if_source_newer"] + + if overwrite_existing_files not in allowed_overwrite: + utils.log_and_raise_error( + f"`overwrite_existing_files` not " + f"recognised, must be one of: " + f"{allowed_overwrite}", + ValueError, + ) + + return { + "overwrite_existing_files": overwrite_existing_files, + "show_transfer_progress": True, + "transfer_verbosity": "vv", + "dry_run": dry_run, + } diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index bc419d951..1185b73ce 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -791,7 +791,7 @@ def _transfer_specific_file_or_folder( upload_or_download, top_level_folder, include_list, - self.cfg.make_rclone_transfer_options( + self.cfg.rclone.make_rclone_transfer_options( overwrite_existing_files, dry_run ), ) @@ -845,7 +845,7 @@ def setup_ssh_connection(self) -> None: utils.log_and_message( f"Your SSH key will be stored in the rclone config at:\n " - f"{self.cfg.get_rclone_config_filepath()}.\n\n" + f"{self.cfg.rclone.get_rclone_config_filepath()}.\n\n" ) if not self.cfg.get_rclone_has_password(): @@ -897,7 +897,7 @@ def setup_gdrive_connection(self) -> None: config_token = gdrive.prompt_and_get_config_token( self.cfg, gdrive_client_secret, - self.cfg.get_rclone_config_name("gdrive"), + self.cfg.rclone.get_rclone_config_name("gdrive"), log=True, ) else: @@ -987,7 +987,7 @@ def _try_set_rclone_password( try: self.set_rclone_password() except Exception as e: - config_path = self.cfg.get_rclone_config_filepath() + config_path = self.cfg.rclone.get_rclone_config_filepath() utils.log_and_raise_error( f"{str(e)}\n" @@ -1010,7 +1010,7 @@ def set_rclone_password(self): rclone_password.run_rclone_config_encrypt(self.cfg) - self.cfg.set_rclone_has_password(True) + self.cfg.rclone.set_rclone_has_password(True) def remove_rclone_password(self): """""" @@ -1022,7 +1022,7 @@ def remove_rclone_password(self): rclone_password.remove_rclone_password(self.cfg) - self.cfg.set_rclone_has_password(False) + self.cfg.rclone.set_rclone_has_password(False) # ------------------------------------------------------------------------- # Configs @@ -1671,14 +1671,14 @@ def _setup_rclone_central_ssh_config( ) -> None: rclone.setup_rclone_config_for_ssh( self.cfg, - self.cfg.get_rclone_config_name("ssh"), + self.cfg.rclone.get_rclone_config_name("ssh"), private_key_str, log=log, ) def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( - self.cfg.get_rclone_config_name("local_filesystem"), + self.cfg.rclone.get_rclone_config_name("local_filesystem"), ) def _setup_rclone_gdrive_config( @@ -1688,7 +1688,7 @@ def _setup_rclone_gdrive_config( ) -> subprocess.Popen: return rclone.setup_rclone_config_for_gdrive( self.cfg, - self.cfg.get_rclone_config_name("gdrive"), + self.cfg.rclone.get_rclone_config_name("gdrive"), gdrive_client_secret, config_token, ) @@ -1698,7 +1698,7 @@ def _setup_rclone_aws_config( ) -> None: rclone.setup_rclone_config_for_aws( self.cfg, - self.cfg.get_rclone_config_name("aws"), + self.cfg.rclone.get_rclone_config_name("aws"), aws_secret_access_key, log=log, ) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 5e5bedbf2..4456eb4a9 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -571,7 +571,7 @@ def get_rclone_message_for_gdrive_without_browser( output = gdrive.preliminary_for_setup_without_browser( self.project.cfg, gdrive_client_secret, - self.project.cfg.get_rclone_config_name("gdrive"), + self.project.cfg.rclone.get_rclone_config_name("gdrive"), log=False, ) return True, output diff --git a/datashuttle/utils/aws.py b/datashuttle/utils/aws.py index 9bf1a27cc..ba08183cf 100644 --- a/datashuttle/utils/aws.py +++ b/datashuttle/utils/aws.py @@ -12,7 +12,7 @@ def check_if_aws_bucket_exists(cfg: Configs) -> bool: """ output = rclone.call_rclone_for_central_connection( cfg, - f"lsjson {cfg.get_rclone_config_name()}: {rclone.get_config_arg(cfg)}", + f"lsjson {cfg.rclone.get_rclone_config_name()}: {rclone.get_config_arg(cfg)}", pipe_std=True, ) diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index c21d39bda..983c1e1c0 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -99,7 +99,7 @@ def __init__( self.__upload_or_download, self.__top_level_folder, include_list, - cfg.make_rclone_transfer_options( + cfg.rclone.make_rclone_transfer_options( overwrite_existing_files, dry_run ), ) diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 2b31e64b9..7f2e2d11a 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -695,7 +695,7 @@ def search_central_via_connection( If `True`, return the full filepath, otherwise return only the folder/file name. """ - rclone_config_name = cfg.get_rclone_config_name( + rclone_config_name = cfg.rclone.get_rclone_config_name( cfg["connection_method"] ) # TODO: this is not good because we get the config name here and in get_config_arg diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index e227133d1..74b56e1c0 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -209,7 +209,7 @@ def setup_rclone_config_for_local_filesystem( ---------- rclone_config_name canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + datashuttle.cfg.rclone.get_rclone_config_name() log whether to log, if True logger must already be initialised. @@ -240,7 +240,7 @@ def setup_rclone_config_for_ssh( rclone_config_name canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + datashuttle.cfg.rclone.get_rclone_config_name() private_key_str PEM encoded ssh private key to pass to RClone. @@ -256,7 +256,7 @@ def setup_rclone_config_for_ssh( ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() - cfg.set_rclone_has_password(False) + cfg.rclone.set_rclone_has_password(False) command = ( f"config create " @@ -282,12 +282,12 @@ def get_config_path(): def get_full_config_filepath(cfg: Configs) -> Path: - return get_config_path() / f"{cfg.get_rclone_config_name()}.conf" + return get_config_path() / f"{cfg.rclone.get_rclone_config_name()}.conf" def get_config_arg(cfg): """TODO PLACEHOLDER.""" - cfg.get_rclone_config_name() # pass this? handle better... + cfg.rclone.get_rclone_config_name() # pass this? handle better... if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: return f'--config "{get_full_config_filepath(cfg)}"' @@ -327,7 +327,7 @@ def setup_rclone_config_for_gdrive( rclone_config_name Canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + datashuttle.cfg.rclone.get_rclone_config_name() gdrive_client_secret Google Drive client secret, mandatory when using a Google Drive client. @@ -358,7 +358,7 @@ def setup_rclone_config_for_gdrive( ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() - cfg.set_rclone_has_password(False) + cfg.rclone.set_rclone_has_password(False) print("Trying to create gdrive config") process = call_rclone_with_popen_for_central_connection( @@ -392,7 +392,7 @@ def setup_rclone_config_for_aws( rclone_config_name Canonical RClone config name, generated by - datashuttle.cfg.get_rclone_config_name() + datashuttle.cfg.rclone.get_rclone_config_name() aws_secret_access_key The aws secret access key provided by the user. @@ -416,7 +416,7 @@ def setup_rclone_config_for_aws( ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file if rclone_config_filepath.exists(): rclone_config_filepath.unlink() - cfg.set_rclone_has_password(False) + cfg.rclone.set_rclone_has_password(False) output = call_rclone( "config create " @@ -455,7 +455,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: else: tempfile_path = (cfg["central_path"] / filename).as_posix() - config_name = cfg.get_rclone_config_name() + config_name = cfg.rclone.get_rclone_config_name() output = call_rclone_for_central_connection( cfg, @@ -469,7 +469,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: output = call_rclone_for_central_connection( cfg, - f"delete {cfg.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", + f"delete {cfg.rclone.get_rclone_config_name()}:{tempfile_path} {get_config_arg(cfg)}", pipe_std=True, ) if output.returncode != 0: @@ -547,7 +547,7 @@ def transfer_data( rclone_options A list of options to pass to Rclone's copy function. - see `cfg.make_rclone_transfer_options()`. + see `cfg.rclone.make_rclone_transfer_options()`. Returns ------- @@ -578,7 +578,7 @@ def transfer_data( output = call_rclone_through_script_for_central_connection( cfg, f"{rclone_args('copy')} " - f'"{local_filepath}" "{cfg.get_rclone_config_name()}:' + f'"{local_filepath}" "{cfg.rclone.get_rclone_config_name()}:' f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) @@ -586,7 +586,7 @@ def transfer_data( output = call_rclone_through_script_for_central_connection( cfg, f"{rclone_args('copy')} " - f'"{cfg.get_rclone_config_name()}:' + f'"{cfg.rclone.get_rclone_config_name()}:' f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) @@ -696,7 +696,7 @@ def perform_rclone_check( cfg, f"{rclone_args('check')} " f'"{local_filepath}" ' - f'"{cfg.get_rclone_config_name()}:{central_filepath}"' + f'"{cfg.rclone.get_rclone_config_name()}:{central_filepath}"' f"{get_config_arg(cfg)} " f"--combined -", pipe_std=True, diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 2c92d3dcb..20974d1ab 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -92,7 +92,7 @@ def set_password_linux(cfg): ) output = subprocess.run( - f"echo $(openssl rand -base64 40) | pass insert -m {cfg.get_rclone_config_name()}", + f"echo $(openssl rand -base64 40) | pass insert -m {cfg.rclone.get_rclone_config_name()}", shell=True, capture_output=True, text=True, @@ -108,7 +108,7 @@ def set_password_linux(cfg): def set_password_macos(cfg: Configs): """""" output = subprocess.run( - f"security add-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", + f"security add-generic-password -a datashuttle -s {cfg.rclone.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", shell=True, capture_output=True, text=True, @@ -153,18 +153,18 @@ def set_credentials_as_password_command(cfg): elif platform.system() == "Linux": os.environ["RCLONE_PASSWORD_COMMAND"] = ( - f"/usr/bin/pass {cfg.get_rclone_config_name()}" + f"/usr/bin/pass {cfg.rclone.get_rclone_config_name()}" ) elif platform.system() == "Darwin": os.environ["RCLONE_PASSWORD_COMMAND"] = ( - f"/usr/bin/security find-generic-password -a datashuttle -s {cfg.get_rclone_config_name()} -w" + f"/usr/bin/security find-generic-password -a datashuttle -s {cfg.rclone.get_rclone_config_name()} -w" ) def run_rclone_config_encrypt(cfg: Configs): """""" - rclone_config_path = cfg.get_rclone_config_filepath() + rclone_config_path = cfg.rclone.get_rclone_config_filepath() if not rclone_config_path.exists(): connection_method = cfg["connection_method"] @@ -198,7 +198,7 @@ def remove_rclone_password(cfg): """""" set_credentials_as_password_command(cfg) - config_filepath = cfg.get_rclone_config_filepath() + config_filepath = cfg.rclone.get_rclone_config_filepath() output = subprocess.run( rf"rclone config encryption remove --config {config_filepath.as_posix()}", @@ -240,7 +240,7 @@ def get_password_filepath( base_path.mkdir(exist_ok=True, parents=True) - return base_path / f"{cfg.get_rclone_config_name()}.xml" + return base_path / f"{cfg.rclone.rclone.get_rclone_config_name()}.xml" def run_raise_if_fail(command, command_description): diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index bd94b023c..cda422c2a 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -546,3 +546,34 @@ project.setup_aws_connection() Running [](setup_aws_connection()) will require entering your `AWS Secret Access Key` and the setup will be completed. + + +(password-protection)= +### Password protecting your connection credentials + +Datashuttle uses the software `RClone` for all data transfers by default. +RClone stores the credentials for connection by default in an unencrypted configuration file. +This includes: + +- ssh connection: the private SSH key +- Google Drive: some API thing +- Amazon S3: Access key ID and XXX + +By default, these are stored in your home directory which should be secure. However, for an +additional layer of security, it is possible to encrpy the Rclone config file. + +When setting up the connection, datashuttle will offer the option to set the RClone configuration. +This automatically uses the system credential manager: + +Windows : (requires powershell) +macOS : set up +Linux : requires pass + +This means the file is only uncryptable on your local machine or user) CHECK USER. + +TODO: think more about the credentials file... its' stupid to have this itself plain text in datashuttle? + +Despite this layer of security, it is not reccomended to use datashuttle for remote connectivity on +a machine to which you do not have secure access, even with password protection of the RClone config. + +TODO: test if `pass` is not installed on linux that the error is propagated to the TUI properly From b13a9840000b3673cf3bcf3e3b0f5562ab17f1ce Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 7 Oct 2025 16:34:47 +0100 Subject: [PATCH 024/100] Finished refactor of configs. --- datashuttle/configs/rclone_configs.py | 10 +++++++--- datashuttle/datashuttle_class.py | 28 ++++++++++++++++++++------- datashuttle/utils/rclone.py | 20 ++++++++----------- datashuttle/utils/rclone_password.py | 7 +------ 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index e2b1a79ef..1758ea0f1 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -36,7 +36,7 @@ class RCloneConfigs: """ def __init__(self, datashuttle_configs, config_base_path): - self.datashuttle_configs = configs + self.datashuttle_configs = datashuttle_configs self.rclone_password_state_file_path = ( config_base_path / "rclone_ps_state.yaml" ) @@ -75,7 +75,11 @@ def set_rclone_has_password(self, value): Note that this is stored to disk each call (rather than tracked locally) to ensure it is updated live if updated through the Python API while the TUI is also running. """ - assert self["connection_method"] in ["ssh", "aws", "gdrive"] + assert self.datashuttle_configs["connection_method"] in [ + "ssh", + "aws", + "gdrive", + ] rclone_has_password = self.load_rclone_has_password() @@ -103,7 +107,7 @@ def get_rclone_config_name( if connection_method is None: connection_method = self.datashuttle_configs["connection_method"] - return f"central_{self.project_name}_{connection_method}" + return f"central_{self.datashuttle_configs.project_name}_{connection_method}" def get_rclone_config_filepath(self) -> Path: """The full filepath to the rclone `.conf` config file""" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 1185b73ce..00155b2ec 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -828,6 +828,11 @@ def setup_ssh_connection(self) -> None: cluster. Once input, SSH private / public key pair will be setup. """ + if self.cfg["connection_method"] != "ssh": + raise RuntimeError( + "configs `connection_method` must be 'ssh' to set up SSH connection." + ) + self._start_log( "setup-ssh-connection-to-central-server", local_vars=locals() ) @@ -848,7 +853,7 @@ def setup_ssh_connection(self) -> None: f"{self.cfg.rclone.get_rclone_config_filepath()}.\n\n" ) - if not self.cfg.get_rclone_has_password(): + if not self.cfg.rclone.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail( @@ -881,6 +886,11 @@ def setup_gdrive_connection(self) -> None: Next, with the provided credentials, the final setup will be done. This opens up a browser if the user confirmed access to a browser. """ + if self.cfg["connection_method"] != "gdrive": + raise RuntimeError( + "configs `connection_method` must be 'gdrive' to set up Google Drive connection." + ) + self._start_log( "setup-google-drive-connection-to-central-server", local_vars=locals(), @@ -907,14 +917,12 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) - print("got process") - # TODO: do something with stderr stdout here, in general handle errors better! rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( self.cfg, process, log=True ) - if not self.cfg.get_rclone_has_password(): + if not self.cfg.rclone.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -938,6 +946,12 @@ def setup_aws_connection(self) -> None: Next, with the provided credentials, the final connection setup will be done. """ + if self.cfg["connection_method"] != "aws": + raise RuntimeError( + "configs `connection_method` must be 'aws' to " + "set up Amazon Web Services S3 Bucket connection." + ) + self._start_log( "setup-aws-connection-to-central-server", local_vars=locals(), @@ -947,7 +961,7 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) - if not self.cfg.get_rclone_has_password(): + if not self.cfg.rclone.get_rclone_has_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -1002,7 +1016,7 @@ def _try_set_rclone_password( def set_rclone_password(self): """""" - if self.cfg.get_rclone_has_password(): + if self.cfg.rclone.get_rclone_has_password(): raise RuntimeError( "This config file already has a password set. " "First, use `remove_rclone_password` to remove it." @@ -1014,7 +1028,7 @@ def set_rclone_password(self): def remove_rclone_password(self): """""" - if not self.cfg.get_rclone_has_password(): + if not self.cfg.rclone.get_rclone_has_password(): raise RuntimeError( f"The config for the current connection method: {self.cfg['connection_method']} " f"does not have a password. Cannot remove." diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 74b56e1c0..d488e5882 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -126,7 +126,7 @@ def call_rclone_with_popen_for_central_connection( process explicitly. """ - # if cfg.get_rclone_has_password(): + # if cfg.rclone.get_rclone_has_password(): # rclone_password.set_credentials_as_password_command(cfg) command = "rclone " + command @@ -165,20 +165,16 @@ def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ - set_password = cfg.get_rclone_has_password() - print("set_password", set_password) + set_password = cfg.rclone.get_rclone_has_password() if set_password: rclone_password.set_credentials_as_password_command(cfg) - print("os env", os.environ) - results = lambda_func() if set_password: rclone_password.remove_credentials_as_password_command() - print("res") return results @@ -360,8 +356,7 @@ def setup_rclone_config_for_gdrive( rclone_config_filepath.unlink() cfg.rclone.set_rclone_has_password(False) - print("Trying to create gdrive config") - process = call_rclone_with_popen_for_central_connection( + command = ( f"config create " f"{rclone_config_name} " f"drive " @@ -370,9 +365,11 @@ def setup_rclone_config_for_gdrive( f"scope drive " f"root_folder_id {cfg['gdrive_root_folder_id']} " f"{extra_args} " - f"{get_config_arg(cfg)}", + f"{get_config_arg(cfg)}" ) - print("Created gdrive config") + + process = call_rclone_with_popen_for_central_connection(command) + return process @@ -590,8 +587,7 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.get_rclone_has_password(): - print("REMOVED") + if cfg.rclone.get_rclone_has_password(): rclone_password.remove_credentials_as_password_command() return output diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 20974d1ab..fb131f744 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -43,7 +43,6 @@ def set_password_windows(cfg: Configs): "(ConvertTo-SecureString ([System.Web.Security.Membership]::GeneratePassword(40,10)) -AsPlainText -Force) " f"| Export-Clixml -LiteralPath '{password_filepath}'" ) - print("set password cmd windows: ", ps_cmd) output = subprocess.run( [shell, "-NoProfile", "-Command", ps_cmd], @@ -127,8 +126,6 @@ def set_credentials_as_password_command(cfg): if platform.system() == "Windows": password_filepath = get_password_filepath(cfg) - print("password_filepath ", password_filepath) - assert password_filepath.exists(), ( "Critical error: password file not found when setting password command." ) @@ -147,8 +144,6 @@ def set_credentials_as_password_command(cfg): f"(Import-Clixml -LiteralPath '{password_filepath}' ).Password)))\"" ) - print("setting rclone cmd: ", cmd) - os.environ["RCLONE_PASSWORD_COMMAND"] = cmd elif platform.system() == "Linux": @@ -240,7 +235,7 @@ def get_password_filepath( base_path.mkdir(exist_ok=True, parents=True) - return base_path / f"{cfg.rclone.rclone.get_rclone_config_name()}.xml" + return base_path / f"{cfg.rclone.get_rclone_config_name()}.xml" def run_raise_if_fail(command, command_description): From 190f324da164745077930ef016ee379b5470f461 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 7 Oct 2025 16:38:36 +0100 Subject: [PATCH 025/100] Add doc to setup_gdrive. --- datashuttle/tui/screens/setup_gdrive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 45969dc2f..580d6a82f 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -25,8 +25,10 @@ class SetupGdriveScreen(ModalScreen): If the config contains a "gdrive_client_id", the user is prompted to enter a client secret. If the user has access to a browser, a Google Drive - authentication page will open. Otherwise, the user is asked to run an rclone command + authentication page will open. Otherwise, the user is asked to run a rclone command and input a config token. + + """ def __init__(self, interface: Interface) -> None: From c9a8fb27f55edb766a128df86e8c97bbd8beb9f9 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 7 Oct 2025 18:30:23 +0100 Subject: [PATCH 026/100] Handling TODOs. --- datashuttle/configs/rclone_configs.py | 2 +- datashuttle/datashuttle_class.py | 24 +------- datashuttle/tui/screens/setup_aws.py | 10 ++-- datashuttle/tui/screens/setup_gdrive.py | 6 +- datashuttle/utils/rclone.py | 78 ++++++++++++------------- datashuttle/utils/rclone_password.py | 10 +++- 6 files changed, 54 insertions(+), 76 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 1758ea0f1..35689c899 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -109,7 +109,7 @@ def get_rclone_config_name( return f"central_{self.datashuttle_configs.project_name}_{connection_method}" - def get_rclone_config_filepath(self) -> Path: + def get_rclone_central_connection_config_filepath(self) -> Path: """The full filepath to the rclone `.conf` config file""" return ( canonical_folders.get_rclone_config_base_path() diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 00155b2ec..c3dfe4bd0 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -802,17 +802,6 @@ def _transfer_specific_file_or_folder( # SSH # ------------------------------------------------------------------------- - # TODO: MAKE MORE NOTES ON HOW THE GDRIVE WORKER IS THE BEST MODEL - # IT MUST BE DONE WILL NOT WORK WITHOUT - - # TODO: this is going to be a massive pain because old config files will not work - # will need to re-set up all connections - # this can just be a breaking change, but will have to handle error nicely - # We could just move it from the config file, then show a warning - - # TODO: need the cancel button on tui in case we close the google window - # THEN we can hide it while we make the connection to check - @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: @@ -850,7 +839,7 @@ def setup_ssh_connection(self) -> None: utils.log_and_message( f"Your SSH key will be stored in the rclone config at:\n " - f"{self.cfg.rclone.get_rclone_config_filepath()}.\n\n" + f"{self.cfg.rclone.get_rclone_central_connection_config_filepath()}.\n\n" ) if not self.cfg.rclone.get_rclone_has_password(): @@ -917,7 +906,6 @@ def setup_gdrive_connection(self) -> None: gdrive_client_secret, config_token ) - # TODO: do something with stderr stdout here, in general handle errors better! rclone.await_call_rclone_with_popen_for_central_connection_raise_on_fail( self.cfg, process, log=True ) @@ -975,11 +963,7 @@ def setup_aws_connection(self) -> None: # Rclone config password # ------------------------------------------------------------------------- - # TODO: LOAD AND SAVE CONFIG FILE ON EACH USE!! - - def _try_set_rclone_password( - self, ask_for_input=True - ): # TODO: handle this better + def _try_set_rclone_password(self, ask_for_input=True): """""" if ask_for_input: pass_type = { @@ -1001,7 +985,7 @@ def _try_set_rclone_password( try: self.set_rclone_password() except Exception as e: - config_path = self.cfg.rclone.get_rclone_config_filepath() + config_path = self.cfg.rclone.get_rclone_central_connection_config_filepath() utils.log_and_raise_error( f"{str(e)}\n" @@ -1012,8 +996,6 @@ def _try_set_rclone_password( utils.log_and_message("Password set successfully") - # TODO: REMOVE from (e) just print (e) - def set_rclone_password(self): """""" if self.cfg.rclone.get_rclone_has_password(): diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 88a1abb31..d629080eb 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -71,7 +71,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss() elif event.button.id == "setup_aws_ok_button": - if self.stage == "init": self.prompt_user_for_aws_secret_access_key() @@ -79,7 +78,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.use_secret_access_key_to_setup_aws_connection() elif self.stage == "ask_password": - self.ask_for_password() + self.set_password() elif self.stage == "finished": self.dismiss() @@ -97,7 +96,7 @@ def prompt_user_for_aws_secret_access_key(self) -> None: def use_secret_access_key_to_setup_aws_connection(self) -> None: """Set up the AWS connection and failure. If success, move onto the - password screen. + password screen. """ secret_access_key = self.query_one( @@ -118,7 +117,7 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: self.stage = "ask_password" else: message = ( - f"AWS setup failed. Please check your configs and secret access key" # TODO: check this + f"AWS setup failed. Please check your configs and secret access key" f"\n\n Traceback: {output}" ) self.query_one( @@ -128,8 +127,7 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: self.query_one("#setup_aws_ok_button").label = "Retry" self.query_one("#setup_aws_messagebox_message").update(message) - # TODO: this is a direct copy - def ask_for_password(self): # TODO: CHANGE NAME + def set_password(self): """""" success, output = self.interface.try_setup_rclone_password() diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 580d6a82f..6d3991f8b 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -295,8 +295,8 @@ async def setup_gdrive_connection_and_update_ui( try: widget = self.query_one(id) await widget.remove() - except BaseException: - pass # TODO + except textual.errors.NoMatches: + pass else: self.input_box.disabled = False self.enter_button.disabled = False @@ -324,6 +324,7 @@ def setup_gdrive_connection( # UI Update Methods # ---------------------------------------------------------------------------------- + # TODO: REFACTOR THIS IN GENERAL BIT CONFUSING NOW def set_finish_page(self) -> None: # TODO: NOW DUPLCIATE """Show the final screen after successful set up.""" message = "Setup Complete!" @@ -345,7 +346,6 @@ def show_password_screen(self): yes_button, no_button ) - # TODO: DIRECT COPY def set_password(self): """""" success, output = self.interface.try_setup_rclone_password() diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d488e5882..d8c3499c4 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -156,7 +156,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( utils.log_and_raise_error(stderr.decode("utf-8"), ConnectionError) if log: - log_rclone_config_output() + log_rclone_config_output(cfg) return stdout, stderr @@ -214,7 +214,7 @@ def setup_rclone_config_for_local_filesystem( call_rclone(f"config create {rclone_config_name} local", pipe_std=True) if log: - log_rclone_config_output() + log_rclone_config_output(cfg) def setup_rclone_config_for_ssh( @@ -247,12 +247,7 @@ def setup_rclone_config_for_ssh( """ key_escaped = private_key_str.replace("\n", "\\n") - rclone_config_filepath = get_full_config_filepath( - cfg - ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) + delete_existing_rclone_config_file(cfg) command = ( f"config create " @@ -267,26 +262,37 @@ def setup_rclone_config_for_ssh( call_rclone(command, pipe_std=True) if log: - log_rclone_config_output() + log_rclone_config_output(cfg) -def get_config_path(): - """TODO PLACEHOLDER.""" +def get_full_config_filepath(cfg: Configs) -> Path: + """ """ return ( - Path().home() / "AppData" / "Roaming" / "rclone" - ) # # "$HOME/.config/rclone/rclone.conf") + canonical_folders.get_rclone_config_base_path() + / f"{cfg.rclone.get_rclone_config_name()}.conf" + ) -def get_full_config_filepath(cfg: Configs) -> Path: - return get_config_path() / f"{cfg.rclone.get_rclone_config_name()}.conf" +def delete_existing_rclone_config_file(cfg: Configs): + """ """ + rclone_config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if delete_existing_file: + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.rclone.set_rclone_has_password(False) def get_config_arg(cfg): """TODO PLACEHOLDER.""" - cfg.rclone.get_rclone_config_name() # pass this? handle better... + rclone_config_path = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: - return f'--config "{get_full_config_filepath(cfg)}"' + return f'--config "{rclone_config_path}"' else: return "" @@ -349,12 +355,7 @@ def setup_rclone_config_for_gdrive( else "" ) - rclone_config_filepath = get_full_config_filepath( - cfg - ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) + delete_existing_rclone_config_file(cfg) command = ( f"config create " @@ -408,12 +409,7 @@ def setup_rclone_config_for_aws( else f" location_constraint {aws_region}" ) - rclone_config_filepath = get_full_config_filepath( - cfg - ) # TODO: do this for everything TODO: maybe this config file can be created before setup in case of old file - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) + delete_existing_rclone_config_file(cfg) output = call_rclone( "config create " @@ -433,7 +429,7 @@ def setup_rclone_config_for_aws( ) if log: - log_rclone_config_output() + log_rclone_config_output(cfg) def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: @@ -475,12 +471,17 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: ) -def log_rclone_config_output() -> None: # TODO: remove or update this +def log_rclone_config_output(cfg: Configs) -> None: """Log the output from creating Rclone config.""" - output = call_rclone("config file", pipe_std=True) - utils.log( - f"Successfully created rclone config. {output.stdout.decode('utf-8')}" - ) + if cfg["connection_method"] in ["aws", "ssh", "gdrive"]: + config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + else: + output = call_rclone("config file", pipe_std=True) + config_filepath = output.stdout.decode("utf-8") + + utils.log(f"Successfully created rclone config. {config_filepath}") def prompt_rclone_download_if_does_not_exist() -> None: @@ -564,13 +565,6 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) - # if cfg.backend_has_password[cfg["connection_method"]]: # TODO: one getter - # print("SET") - # config_filepath = rclone_password.get_password_filepath( - # cfg - # ) # TODO: ONE FUNCTION OR INCORPORATE INTO SINGLE FUNCTION - # rclone_password.set_credentials_as_password_command(config_filepath) - if upload_or_download == "upload": output = call_rclone_through_script_for_central_connection( cfg, diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index fb131f744..5413f704c 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -82,7 +82,7 @@ def set_password_linux(cfg): raise RuntimeError( "Password store is not initialized. " "Run `pass init ` before using `pass`." - ) from e + ) else: raise RuntimeError( f"\n--- STDOUT ---\n{output.stdout}", @@ -159,7 +159,9 @@ def set_credentials_as_password_command(cfg): def run_rclone_config_encrypt(cfg: Configs): """""" - rclone_config_path = cfg.rclone.get_rclone_config_filepath() + rclone_config_path = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) if not rclone_config_path.exists(): connection_method = cfg["connection_method"] @@ -193,7 +195,9 @@ def remove_rclone_password(cfg): """""" set_credentials_as_password_command(cfg) - config_filepath = cfg.rclone.get_rclone_config_filepath() + config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) output = subprocess.run( rf"rclone config encryption remove --config {config_filepath.as_posix()}", From 71fed57a68ba7cc736ac05a242dc21081823412c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 7 Oct 2025 18:57:37 +0100 Subject: [PATCH 027/100] Refactor gdrive page. --- datashuttle/tui/screens/setup_gdrive.py | 87 ++++++++++++------------- 1 file changed, 40 insertions(+), 47 deletions(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 6d3991f8b..96e4405fc 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -27,8 +27,6 @@ class SetupGdriveScreen(ModalScreen): to enter a client secret. If the user has access to a browser, a Google Drive authentication page will open. Otherwise, the user is asked to run a rclone command and input a config token. - - """ def __init__(self, interface: Interface) -> None: @@ -36,7 +34,7 @@ def __init__(self, interface: Interface) -> None: super(SetupGdriveScreen, self).__init__() self.interface = interface - self.stage: int = 0 + self.no_browser_stage: None | str = "show_command_to_generate_code" self.setup_worker: Worker | None = None self.is_browser_available: bool = True self.gdrive_client_secret: Optional[str] = None @@ -73,20 +71,24 @@ def on_button_pressed(self, event: Button.Pressed) -> None: This dialog window operates using 6 buttons: - 1) "ok" button : Starts the connection setup process. + 1) `setup_gdrive_ok_button` : Starts the connection setup process. - 2) "yes" button : A "yes" answer to the availability of browser question. On click, - if "gdrive_client_id" is present in configs, the user is asked for client secret + 2) `setup_gdrive_has_browser_yes_button` : A "yes" answer to the availability of browser question. + On click, if "gdrive_client_id" is present in configs, the user is asked for client secret and proceeds to a browser authentication. - 3) "no" button : A "no" answer to the availability of browser question. On click, + 3) `setup_gdrive_no_button` : A "no" answer to the availability of browser question. On click, prompts the user to enter a config token by running an rclone command. - 4) "enter" button : To enter the client secret or config token. + 4) `setup_gdrive_no_browser_enter_button` : To enter the client secret or config token. + + 5) `setup_gdrive_set_password_yes_button` : To set a password on the RClone config file + + 6) `setup_gdrive_set_password_no_button` : To skip setting a password on the RClone config file - 5) "finish" button : To finish the setup. + 7) `setup_gdrive_finish_button` button : To finish the setup. - 6) "cancel" button : To cancel the setup at any step before completion. + 8) "`setup_gdrive_cancel_button` : To cancel the setup at any step before completion. """ if ( event.button.id == "setup_gdrive_cancel_button" @@ -94,7 +96,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ): # see setup_gdrive_connection_and_update_ui() if self.setup_worker and self.setup_worker.is_running: - self.setup_worker.cancel() # fix + self.setup_worker.cancel() self.interface.terminate_gdrive_setup() self.dismiss() @@ -107,27 +109,29 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.ask_user_for_browser() elif event.button.id == "setup_gdrive_has_browser_yes_button": - self.remove_yes_no_browser_buttons() + self.query_one("#setup_gdrive_has_browser_yes_button").remove() + self.query_one("#setup_gdrive_no_button").remove() self.open_browser_and_setup_gdrive_connection( self.gdrive_client_secret ) elif event.button.id == "setup_gdrive_no_button": self.is_browser_available = False - self.remove_yes_no_browser_buttons() + self.query_one("#setup_gdrive_has_browser_yes_button").remove() + self.query_one("#setup_gdrive_no_button").remove() self.prompt_user_for_config_token() elif event.button.id == "setup_gdrive_no_browser_enter_button": if ( self.interface.project.cfg["gdrive_client_id"] - and self.stage == 0 + and self.no_browser_stage == "show_command_to_generate_code" ): self.gdrive_client_secret = ( self.input_box.value.strip() if self.input_box.value.strip() else None ) - self.stage += 1 + self.no_browser_stage = "setup_with_code" self.ask_user_for_browser() else: config_token = ( @@ -142,14 +146,11 @@ def on_button_pressed(self, event: Button.Pressed) -> None: elif event.button.id == "setup_gdrive_set_password_yes_button": self.set_password() - elif event.button.id == "setup_gdrive_skip_password_ok_button": - self.query_one("#setup_gdrive_skip_password_ok_button").remove() - self.set_finish_page() - elif event.button.id == "setup_gdrive_set_password_no_button": - self.query_one("#setup_gdrive_set_password_yes_button").remove() - self.query_one("#setup_gdrive_set_password_no_button").remove() - self.set_finish_page() + self.set_finish_page("Setup complete!") + + # Setup the connection (with or without browser) + # ---------------------------------------------------------------------------------- def ask_user_for_browser(self) -> None: """Ask the user if their machine has access to a browser.""" @@ -320,19 +321,8 @@ def setup_gdrive_connection( ) return success, output + # Set password on RClone config # ---------------------------------------------------------------------------------- - # UI Update Methods - # ---------------------------------------------------------------------------------- - - # TODO: REFACTOR THIS IN GENERAL BIT CONFUSING NOW - def set_finish_page(self) -> None: # TODO: NOW DUPLCIATE - """Show the final screen after successful set up.""" - message = "Setup Complete!" - - self.update_message_box_message(message) - self.query_one("#setup_gdrive_buttons_horizontal").mount( - Button("Finish", id="setup_gdrive_finish_button") - ) def show_password_screen(self): """""" @@ -351,20 +341,28 @@ def set_password(self): success, output = self.interface.try_setup_rclone_password() if success: - self.query_one("#setup_gdrive_set_password_yes_button").remove() - self.query_one("#setup_gdrive_set_password_no_button").remove() - - message = "The password was successfully set. Setup complete!" - self.update_message_box_message(message) - self.query_one("#setup_gdrive_buttons_horizontal").mount( - Button("Finish", id="setup_gdrive_finish_button") + self.set_finish_page( + "The password was successfully set. Setup complete!" ) else: message = f"The password set up failed. Exception: {output}" self.update_message_box_message(message) + def set_finish_page(self, message) -> None: + """Show the final screen after successful set up.""" + self.query_one("#setup_gdrive_set_password_yes_button").remove() + self.query_one("#setup_gdrive_set_password_no_button").remove() + + self.update_message_box_message(message) + self.query_one("#setup_gdrive_buttons_horizontal").mount( + Button("Finish", id="setup_gdrive_finish_button") + ) + + # UI Update Methods + # ---------------------------------------------------------------------------------- + def display_failed(self, output) -> None: - """Update the message box indicating the set up failed.""" + """Update the message box indicating the set-up failed.""" message = ( f"Google Drive setup failed. Please check your credentials" f"\n\n Traceback: {output}" @@ -389,8 +387,3 @@ def mount_input_box_before_buttons( ) self.input_box.visible = True self.input_box.value = "" - - def remove_yes_no_browser_buttons(self) -> None: - """Remove yes and no buttons.""" - self.query_one("#setup_gdrive_has_browser_yes_button").remove() - self.query_one("#setup_gdrive_no_button").remove() From c743859cd2f08ed9ede9082a1b964e86b5450187 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 8 Oct 2025 17:30:43 +0100 Subject: [PATCH 028/100] Fix variable. --- datashuttle/utils/rclone.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d8c3499c4..46823423b 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -279,10 +279,9 @@ def delete_existing_rclone_config_file(cfg: Configs): cfg.rclone.get_rclone_central_connection_config_filepath() ) - if delete_existing_file: - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.rclone.set_rclone_has_password(False) def get_config_arg(cfg): From 37e78f701cfc587a34ac04d9fe891c8b2c050162 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 8 Oct 2025 17:45:40 +0100 Subject: [PATCH 029/100] Extend the gdrive setup message. --- datashuttle/tui/screens/setup_gdrive.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 96e4405fc..6939665ad 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -202,7 +202,12 @@ def open_browser_and_setup_gdrive_connection( The connection setup is asynchronous so that the user is able to cancel the setup if anything goes wrong without quitting datashuttle altogether. """ - message = "Please authenticate through browser (it should open automatically)." + message = ( + "Please authenticate through your browser (it should open automatically).\n\n" + "It may take a moment for the connection to register after you confirm in the browser.\n" + "Only click 'Cancel' if you are sure you want to stop the setup." + ) + self.update_message_box_message(message) task = asyncio.create_task( From 73995ebc365244f035ec25ca6cb11df4d1627cce Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 8 Oct 2025 17:45:59 +0100 Subject: [PATCH 030/100] Rework `_try_set_rclone_password` --- datashuttle/datashuttle_class.py | 62 ++++++++++++++++---------------- datashuttle/tui/interface.py | 2 +- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index c3dfe4bd0..1f9b00a5b 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -843,7 +843,8 @@ def setup_ssh_connection(self) -> None: ) if not self.cfg.rclone.get_rclone_has_password(): - self._try_set_rclone_password() + if self._ask_user_if_set_password(): + self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail( self.cfg @@ -911,7 +912,8 @@ def setup_gdrive_connection(self) -> None: ) if not self.cfg.rclone.get_rclone_has_password(): - self._try_set_rclone_password() + if self._ask_user_if_set_password(): + self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -950,7 +952,8 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) if not self.cfg.rclone.get_rclone_has_password(): - self._try_set_rclone_password() + if self._ask_user_if_set_password(): + self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) aws.raise_if_bucket_absent(self.cfg) @@ -963,38 +966,37 @@ def setup_aws_connection(self) -> None: # Rclone config password # ------------------------------------------------------------------------- - def _try_set_rclone_password(self, ask_for_input=True): + def _ask_user_if_set_password(self) -> bool: """""" - if ask_for_input: - pass_type = { - "Windows": "Windows credential manager", - "Linux": "the `pass` program", - "Darwin": "macOS inbuild `security`.", - } - - input_ = utils.get_user_input( - f"Would you like to set a password using {pass_type[platform.system()]}.\n" - f"Press 'y' to set password or leave blank to skip." - ) + pass_type = { + "Windows": "Windows credential manager", + "Linux": "the `pass` program", + "Darwin": "macOS inbuild `security`.", + } + + input_ = utils.get_user_input( + f"Would you like to set a password using {pass_type[platform.system()]}.\n" + f"Press 'y' to set password or leave blank to skip." + ) - set_password = input_ == "y" - else: - set_password = True + return input_ == "y" - if set_password: - try: - self.set_rclone_password() - except Exception as e: - config_path = self.cfg.rclone.get_rclone_central_connection_config_filepath() + def _try_set_rclone_password(self): # TODO: use different nomeclature... encrypted not password + """""" + try: + self.set_rclone_password() + except Exception as e: + config_path = self.cfg.rclone.get_rclone_central_connection_config_filepath() - utils.log_and_raise_error( - f"{str(e)}\n" - f"Password set up failed. The config at {config_path} contains the private ssh key without a password.\n" - f"Use set_rclone_password()` to attempt to set the password again (see full error message above). ", - RuntimeError, - ) + utils.log_and_raise_error( + f"{str(e)}\n" + f"Password set up failed.\n" + f"Use set_rclone_password()` to attempt to set the password again (see full error message above).\n" + f"IMPORTANT NOTE: The config at {config_path} does not have a password.\n", + RuntimeError, + ) - utils.log_and_message("Password set successfully") + utils.log_and_message("Password set successfully") def set_rclone_password(self): """""" diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 4456eb4a9..c54a9b25b 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -635,7 +635,7 @@ def setup_aws_connection( def try_setup_rclone_password(self): try: - self.project._try_set_rclone_password(ask_for_input=False) + self.project._try_set_rclone_password() return True, None except BaseException as e: return False, str(e) From 18ffdd548dd4d51b37dccebcbe1cc5f926960127 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 8 Oct 2025 19:14:59 +0100 Subject: [PATCH 031/100] Add TODO fixes. --- datashuttle/configs/rclone_configs.py | 10 ++++ datashuttle/datashuttle_class.py | 2 +- datashuttle/tui/interface.py | 1 + datashuttle/tui/screens/setup_gdrive.py | 7 ++- datashuttle/utils/rclone.py | 76 ++++++++----------------- 5 files changed, 41 insertions(+), 55 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 35689c899..b67d395fc 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -136,3 +136,13 @@ def make_rclone_transfer_options( "transfer_verbosity": "vv", "dry_run": dry_run, } + + def delete_existing_rclone_config_file(self): + """ """ + rclone_config_filepath = ( + self.get_rclone_central_connection_config_filepath() + ) + + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + self.set_rclone_has_password(False) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 1f9b00a5b..de70f1bbd 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1676,7 +1676,7 @@ def _setup_rclone_central_ssh_config( def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( - self.cfg.rclone.get_rclone_config_name("local_filesystem"), + self.cfg, self.cfg.rclone.get_rclone_config_name("local_filesystem"), ) def _setup_rclone_gdrive_config( diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index c54a9b25b..3c39a20f1 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -634,6 +634,7 @@ def setup_aws_connection( # ------------------------------------------------------------------------------------ def try_setup_rclone_password(self): + """""" try: self.project._try_set_rclone_password() return True, None diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 6939665ad..89627c315 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -13,6 +13,7 @@ from textual import work from textual.containers import Container, Horizontal, Vertical from textual.screen import ModalScreen +from textual.css.query import NoMatches from textual.widgets import ( Button, Input, @@ -293,6 +294,10 @@ async def setup_gdrive_connection_and_update_ui( success, output = worker.result if success: self.show_password_screen() + + # This function is called from different screens that + # contain different widgets. Therefore remove all possible + # widgets that may / may not be present on the previous screen. for id in [ "#setup_gdrive_cancel_button", "#setup_gdrive_generic_input_box", @@ -301,7 +306,7 @@ async def setup_gdrive_connection_and_update_ui( try: widget = self.query_one(id) await widget.remove() - except textual.errors.NoMatches: + except NoMatches: pass else: self.input_box.disabled = False diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 46823423b..f4cb18722 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -51,6 +51,7 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: def call_rclone_for_central_connection( cfg, command: str, pipe_std: bool = False ) -> CompletedProcess: + """PLACEHOLDER""" return run_function_that_may_require_central_connection_password( cfg, lambda: call_rclone(command, pipe_std) ) @@ -114,7 +115,7 @@ def call_rclone_through_script_for_central_connection( return output -def call_rclone_with_popen_for_central_connection( +def call_rclone_with_popen( command: str, ) -> subprocess.Popen: """Call rclone using `subprocess.Popen` for control over process termination. @@ -122,18 +123,14 @@ def call_rclone_with_popen_for_central_connection( It is not possible to kill a process while running it using `subprocess.run`. Killing a process might be required when running rclone setup in a thread worker to allow the user to cancel the setup process. In such a case, cancelling the - thread worker alone will not kill the rclone process, so we need to kill thenothe env - - process explicitly. + thread worker alone will not kill the rclone process, so we need to kill the + env process explicitly. """ - # if cfg.rclone.get_rclone_has_password(): - # rclone_password.set_credentials_as_password_command(cfg) - command = "rclone " + command process = subprocess.Popen( shlex.split(command), stdout=subprocess.PIPE, - stderr=subprocess.PIPE, # , env=copy.deepcopy(os.environ) + stderr=subprocess.PIPE, ) return process @@ -184,6 +181,7 @@ def run_function_that_may_require_central_connection_password( def setup_rclone_config_for_local_filesystem( + cfg: Configs, rclone_config_name: str, log: bool = True, ) -> None: @@ -247,7 +245,7 @@ def setup_rclone_config_for_ssh( """ key_escaped = private_key_str.replace("\n", "\\n") - delete_existing_rclone_config_file(cfg) + cfg.rclone.delete_existing_rclone_config_file() command = ( f"config create " @@ -265,46 +263,6 @@ def setup_rclone_config_for_ssh( log_rclone_config_output(cfg) -def get_full_config_filepath(cfg: Configs) -> Path: - """ """ - return ( - canonical_folders.get_rclone_config_base_path() - / f"{cfg.rclone.get_rclone_config_name()}.conf" - ) - - -def delete_existing_rclone_config_file(cfg: Configs): - """ """ - rclone_config_filepath = ( - cfg.rclone.get_rclone_central_connection_config_filepath() - ) - - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) - - -def get_config_arg(cfg): - """TODO PLACEHOLDER.""" - rclone_config_path = ( - cfg.rclone.get_rclone_central_connection_config_filepath() - ) - - if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: - return f'--config "{rclone_config_path}"' - else: - return "" - - -def set_password(cfg, password: str): - subprocess.run( - f"rclone config encryption set {get_config_arg(cfg)}", text=True - ) - - -# def remove_password(): - - def setup_rclone_config_for_gdrive( cfg: Configs, rclone_config_name: str, @@ -313,7 +271,7 @@ def setup_rclone_config_for_gdrive( ) -> subprocess.Popen: """Set up rclone config for connections to Google Drive. - This function uses `call_rclone_with_popen_for_central_connection` instead of `call_rclone`. This + This function uses `call_rclone_with_popen` instead of `call_rclone`. This is done to have more control over the setup process in case the user wishes to cancel the setup. Since the rclone setup for google drive uses a local web server for authentication to google drive, the running process must be killed before the @@ -354,7 +312,7 @@ def setup_rclone_config_for_gdrive( else "" ) - delete_existing_rclone_config_file(cfg) + cfg.rclone.delete_existing_rclone_config_file() command = ( f"config create " @@ -368,7 +326,7 @@ def setup_rclone_config_for_gdrive( f"{get_config_arg(cfg)}" ) - process = call_rclone_with_popen_for_central_connection(command) + process = call_rclone_with_popen(command) return process @@ -408,7 +366,7 @@ def setup_rclone_config_for_aws( else f" location_constraint {aws_region}" ) - delete_existing_rclone_config_file(cfg) + cfg.rclone.delete_existing_rclone_config_file() output = call_rclone( "config create " @@ -431,6 +389,18 @@ def setup_rclone_config_for_aws( log_rclone_config_output(cfg) +def get_config_arg(cfg: Configs) -> str: + """TODO PLACEHOLDER.""" + rclone_config_path = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: + return f'--config "{rclone_config_path}"' + else: + return "" + + def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: """Check for a successful connection by creating a file on the remote. From 611a4402f9c503b4475abf4f89fee7415fd39405 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:46:32 +0000 Subject: [PATCH 032/100] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datashuttle/datashuttle_class.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index de70f1bbd..3797b73a5 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -981,12 +981,16 @@ def _ask_user_if_set_password(self) -> bool: return input_ == "y" - def _try_set_rclone_password(self): # TODO: use different nomeclature... encrypted not password + def _try_set_rclone_password( + self, + ): # TODO: use different nomeclature... encrypted not password """""" try: self.set_rclone_password() except Exception as e: - config_path = self.cfg.rclone.get_rclone_central_connection_config_filepath() + config_path = ( + self.cfg.rclone.get_rclone_central_connection_config_filepath() + ) utils.log_and_raise_error( f"{str(e)}\n" From 5cf30aab81923b8e4c384f7bc44d5974939aac0a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 9 Oct 2025 15:18:19 +0100 Subject: [PATCH 033/100] Fix message asking to use security, add try / except to async --- datashuttle/datashuttle_class.py | 19 ++---- datashuttle/tui/screens/setup_aws.py | 4 +- datashuttle/tui/screens/setup_gdrive.py | 77 +++++++++++++------------ datashuttle/tui/screens/setup_ssh.py | 5 +- datashuttle/utils/rclone_password.py | 25 ++++++++ 5 files changed, 77 insertions(+), 53 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 3797b73a5..79938f391 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -4,7 +4,6 @@ import glob import json import os -import platform import shutil from pathlib import Path from typing import ( @@ -839,11 +838,11 @@ def setup_ssh_connection(self) -> None: utils.log_and_message( f"Your SSH key will be stored in the rclone config at:\n " - f"{self.cfg.rclone.get_rclone_central_connection_config_filepath()}.\n\n" + f"{self.cfg.rclone.get_rclone_central_connection_config_filepath()}.\n" ) if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_set_password(): + if self._ask_user_if_they_want_rclone_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail( @@ -912,7 +911,7 @@ def setup_gdrive_connection(self) -> None: ) if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_set_password(): + if self._ask_user_if_they_want_rclone_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -952,7 +951,7 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_set_password(): + if self._ask_user_if_they_want_rclone_password(): self._try_set_rclone_password() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -966,16 +965,10 @@ def setup_aws_connection(self) -> None: # Rclone config password # ------------------------------------------------------------------------- - def _ask_user_if_set_password(self) -> bool: + def _ask_user_if_they_want_rclone_password(self) -> bool: """""" - pass_type = { - "Windows": "Windows credential manager", - "Linux": "the `pass` program", - "Darwin": "macOS inbuild `security`.", - } - input_ = utils.get_user_input( - f"Would you like to set a password using {pass_type[platform.system()]}.\n" + f"{rclone_password.get_password_explanation_message(self.cfg)}\n" f"Press 'y' to set password or leave blank to skip." ) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index d629080eb..e39c859c7 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -108,7 +108,9 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: ) if success: - message = "Would you like to set a password?" + message = ( + f"{rclone_password.get_password_explanation_message(self.cfg)}" + ) self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").remove() diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 89627c315..a6f0938f7 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import traceback from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: @@ -12,6 +13,7 @@ from textual import work from textual.containers import Container, Horizontal, Vertical +from textual.dom import NoMatches from textual.screen import ModalScreen from textual.css.query import NoMatches from textual.widgets import ( @@ -20,6 +22,8 @@ Static, ) +from datashuttle.utils import rclone_password + class SetupGdriveScreen(ModalScreen): """Dialog window that sets up a Google Drive connection. @@ -205,22 +209,17 @@ def open_browser_and_setup_gdrive_connection( """ message = ( "Please authenticate through your browser (it should open automatically).\n\n" - "It may take a moment for the connection to register after you confirm in the browser.\n" - "Only click 'Cancel' if you are sure you want to stop the setup." + "It may take a moment for the connection to register after you confirm in the browser.\n\n" ) self.update_message_box_message(message) - task = asyncio.create_task( + self._task = asyncio.create_task( self.setup_gdrive_connection_and_update_ui( gdrive_client_secret=gdrive_client_secret ), name="setup_gdrive_connection_with_browser_task", ) - task.add_done_callback(self.handle_task_result) - - def handle_task_result(self, task): - task.result() def prompt_user_for_config_token(self) -> None: """Prompt the user for the rclone config token for Google Drive setup.""" @@ -253,7 +252,7 @@ def setup_gdrive_connection_using_config_token( message = "Setting up connection..." self.update_message_box_message(message) - asyncio.create_task( + self._task = asyncio.create_task( self.setup_gdrive_connection_and_update_ui( gdrive_client_secret=gdrive_client_secret, config_token=config_token, @@ -284,34 +283,40 @@ async def setup_gdrive_connection_and_update_ui( self.input_box.disabled = True self.enter_button.disabled = True - worker = self.setup_gdrive_connection( - gdrive_client_secret, config_token - ) - self.setup_worker = worker - if worker.is_running: - await worker.wait() + try: + worker = self.setup_gdrive_connection( + gdrive_client_secret, config_token + ) + self.setup_worker = worker + if worker.is_running: + await worker.wait() + + success, output = worker.result + if success: + self.show_password_screen() + # This function is called from different screens that + # contain different widgets. Therefore, remove all possible + # widgets that may / may not be present on the previous screen. + for id in [ + "#setup_gdrive_cancel_button", + "#setup_gdrive_generic_input_box", + "#setup_gdrive_no_browser_enter_button", + ]: + try: + widget = self.query_one(id) + await widget.remove() + except NoMatches: + pass + else: + self.input_box.disabled = False + self.enter_button.disabled = False + self.display_failed(output) - success, output = worker.result - if success: - self.show_password_screen() - - # This function is called from different screens that - # contain different widgets. Therefore remove all possible - # widgets that may / may not be present on the previous screen. - for id in [ - "#setup_gdrive_cancel_button", - "#setup_gdrive_generic_input_box", - "#setup_gdrive_no_browser_enter_button", - ]: - try: - widget = self.query_one(id) - await widget.remove() - except NoMatches: - pass - else: - self.input_box.disabled = False - self.enter_button.disabled = False - self.display_failed(output) + except Exception as exc: + tb = "".join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ) + self.display_failed(tb) @work(exclusive=True, thread=True) def setup_gdrive_connection( @@ -336,7 +341,7 @@ def setup_gdrive_connection( def show_password_screen(self): """""" - message = "Would you like to set a password?" + message = f"{rclone_password.get_password_explanation_message(self.interface.project.cfg)}" self.update_message_box_message(message) yes_button = Button("Yes", id="setup_gdrive_set_password_yes_button") diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 163c226d0..368874e84 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -171,9 +171,8 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: if success: message = ( - "Connection set up successfully.\n" - "Would you like to use Windows Credential Manager to set a password on " - "the RClone config file on which your RClone is stored? ." + f"Connection set up successfully.\n" + f"{rclone_password.get_password_explanation_message(self.cfg)}" ) self.query_one("#setup_ssh_ok_button").label = "Yes" self.query_one("#setup_ssh_cancel_button").label = "No" diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_password.py index 5413f704c..0d381717a 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_password.py @@ -255,3 +255,28 @@ def run_raise_if_fail(command, command_description): f"\n--- STDOUT ---\n{output.stdout}\n" f"\n--- STDERR ---\n{output.stderr}\n" ) + + +def get_password_explanation_message( + cfg: Configs, +): # TODO: type when other PR is merged + """""" + system_pass_manager = { + "Windows": "Windows Credential Manager", + "Linux": "the `pass` program", + "Darwin": "macOS built-in `security` tool", + } + + pass_type = { + "ssh": "your private SSH key", + "aws": "your IAM access key ID and seceret access key", + "gdrive": "your Google Drive access token and client secret (if set)", + } + + message = ( + f"By default, RClone stores {pass_type[cfg['connection_method']]} in plain text at the below location:\n\n" + f"{cfg.rclone.get_rclone_central_connection_config_filepath()}\n\n" + f"Would you like to encrypt the RClone config file using {system_pass_manager[platform.system()]}?" + ) + + return message From e53259b353ca108f612f19e4b1954736a1d9d2fa Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 9 Oct 2025 16:35:22 +0100 Subject: [PATCH 034/100] Add backward compatability message. --- datashuttle/__init__.py | 4 ++++ datashuttle/datashuttle_class.py | 4 ++++ datashuttle/utils/rclone.py | 39 ++++++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/datashuttle/__init__.py b/datashuttle/__init__.py index 557521387..3835203aa 100644 --- a/datashuttle/__init__.py +++ b/datashuttle/__init__.py @@ -9,3 +9,7 @@ except PackageNotFoundError: # package is not installed pass + + +def get_datashuttle_version(): + return __version__ diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 79938f391..304af56cf 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1214,6 +1214,10 @@ def get_config_path(self) -> Path: """Return the full path to the DataShuttle config file.""" return self._config_path + @check_configs_set + def get_rclone_central_config_path(self) -> Path: + return rclone.get_rclone_config_filepath(self.cfg) + @check_configs_set def get_configs(self) -> Configs: """Return the datashuttle configs.""" diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index f4cb18722..255e78f25 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -14,6 +14,8 @@ from pathlib import Path from subprocess import CompletedProcess +from packaging import version + from datashuttle.configs import canonical_configs from datashuttle.utils import rclone_password, utils @@ -162,6 +164,24 @@ def run_function_that_may_require_central_connection_password( cfg, lambda_func ): """ """ + from datashuttle import get_datashuttle_version # avoid circular import + + rclone_config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if not rclone_config_filepath.is_file(): + if version.parse(get_datashuttle_version()) <= version.parse("0.7.1"): + raise RuntimeError( + f"The way RClone configs are managed has changed since version v0.7.1\n" + f"Please set up the {cfg['connection_method']} connection again." + ) + else: + raise RuntimeError( + f"An unexpected error occurred. Could not find the rclone config file at: {rclone_config_filepath}\n" + f"Please set up the {cfg['connection_method']} connection again." + ) + set_password = cfg.rclone.get_rclone_has_password() if set_password: @@ -262,6 +282,15 @@ def setup_rclone_config_for_ssh( if log: log_rclone_config_output(cfg) +def delete_existing_rclone_config_file(cfg: Configs): + """ """ + rclone_config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.rclone.set_rclone_has_password(False) def setup_rclone_config_for_gdrive( cfg: Configs, @@ -440,8 +469,8 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: ) -def log_rclone_config_output(cfg: Configs) -> None: - """Log the output from creating Rclone config.""" +def get_rclone_config_filepath(cfg: Configs) -> Path: + """""" if cfg["connection_method"] in ["aws", "ssh", "gdrive"]: config_filepath = ( cfg.rclone.get_rclone_central_connection_config_filepath() @@ -450,6 +479,12 @@ def log_rclone_config_output(cfg: Configs) -> None: output = call_rclone("config file", pipe_std=True) config_filepath = output.stdout.decode("utf-8") + return config_filepath + + +def log_rclone_config_output(cfg: Configs) -> None: + """Log the output from creating Rclone config.""" + config_filepath = get_rclone_config_filepath(cfg) utils.log(f"Successfully created rclone config. {config_filepath}") From 89a6d3c222ee62236eefabd812a37f587abc7ff7 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 9 Oct 2025 17:18:19 +0100 Subject: [PATCH 035/100] minor edit. --- datashuttle/utils/rclone.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 255e78f25..6221912d0 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -196,7 +196,7 @@ def run_function_that_may_require_central_connection_password( # ----------------------------------------------------------------------------- -# Setup +# RClone Configs # ----------------------------------------------------------------------------- @@ -282,16 +282,6 @@ def setup_rclone_config_for_ssh( if log: log_rclone_config_output(cfg) -def delete_existing_rclone_config_file(cfg: Configs): - """ """ - rclone_config_filepath = ( - cfg.rclone.get_rclone_central_connection_config_filepath() - ) - - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) - def setup_rclone_config_for_gdrive( cfg: Configs, rclone_config_name: str, @@ -417,6 +407,16 @@ def setup_rclone_config_for_aws( if log: log_rclone_config_output(cfg) +def delete_existing_rclone_config_file(cfg: Configs): + """ """ + rclone_config_filepath = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if rclone_config_filepath.exists(): + rclone_config_filepath.unlink() + cfg.rclone.set_rclone_has_password(False) + def get_config_arg(cfg: Configs) -> str: """TODO PLACEHOLDER.""" From 3b7915ce4bb6db9f07427b7ce5afe605681c5de5 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 13 Oct 2025 13:40:23 +0100 Subject: [PATCH 036/100] Add docstrings to datashuttle class. --- datashuttle/configs/rclone_configs.py | 6 ++++++ datashuttle/datashuttle_class.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index b67d395fc..324ac8366 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -94,6 +94,12 @@ def get_rclone_has_password( self, ): """Return whether the config file associated with the current `connection_method`.""" + assert self.datashuttle_configs["connection_method"] in [ + "ssh", + "aws", + "gdrive", + ] + rclone_has_password = self.load_rclone_has_password() return rclone_has_password[ diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 304af56cf..89c2cbf4f 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -966,7 +966,7 @@ def setup_aws_connection(self) -> None: # ------------------------------------------------------------------------- def _ask_user_if_they_want_rclone_password(self) -> bool: - """""" + """Get user input to determine if they want to set a password on the rclone config.""" input_ = utils.get_user_input( f"{rclone_password.get_password_explanation_message(self.cfg)}\n" f"Press 'y' to set password or leave blank to skip." @@ -976,8 +976,11 @@ def _ask_user_if_they_want_rclone_password(self) -> bool: def _try_set_rclone_password( self, - ): # TODO: use different nomeclature... encrypted not password - """""" + ) -> None: + """Try to encrypt the rclone config file. + + If it fails, warn the user the config file is unencrypted. + """ try: self.set_rclone_password() except Exception as e: @@ -989,14 +992,14 @@ def _try_set_rclone_password( f"{str(e)}\n" f"Password set up failed.\n" f"Use set_rclone_password()` to attempt to set the password again (see full error message above).\n" - f"IMPORTANT NOTE: The config at {config_path} does not have a password.\n", + f"IMPORTANT: The config at {config_path} is not currently encrpyted.\n", RuntimeError, ) utils.log_and_message("Password set successfully") - def set_rclone_password(self): - """""" + def set_rclone_password(self) -> None: + """Encrypt the rclone config file for the central connection.""" if self.cfg.rclone.get_rclone_has_password(): raise RuntimeError( "This config file already has a password set. " @@ -1007,8 +1010,8 @@ def set_rclone_password(self): self.cfg.rclone.set_rclone_has_password(True) - def remove_rclone_password(self): - """""" + def remove_rclone_password(self) -> None: + """Unencrypt the rclone config file for the central connection.""" if not self.cfg.rclone.get_rclone_has_password(): raise RuntimeError( f"The config for the current connection method: {self.cfg['connection_method']} " From 46ed30ec572fa544dfbdfc6ba7c2b968a738e0f2 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 13 Oct 2025 13:48:05 +0100 Subject: [PATCH 037/100] Start renaming from password to encrpytion. --- datashuttle/datashuttle_class.py | 21 ++++++++++++--------- datashuttle/tui/interface.py | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 89c2cbf4f..2b692b1bc 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -843,7 +843,7 @@ def setup_ssh_connection(self) -> None: if not self.cfg.rclone.get_rclone_has_password(): if self._ask_user_if_they_want_rclone_password(): - self._try_set_rclone_password() + self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail( self.cfg @@ -912,7 +912,7 @@ def setup_gdrive_connection(self) -> None: if not self.cfg.rclone.get_rclone_has_password(): if self._ask_user_if_they_want_rclone_password(): - self._try_set_rclone_password() + self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -952,7 +952,7 @@ def setup_aws_connection(self) -> None: if not self.cfg.rclone.get_rclone_has_password(): if self._ask_user_if_they_want_rclone_password(): - self._try_set_rclone_password() + self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) aws.raise_if_bucket_absent(self.cfg) @@ -974,7 +974,7 @@ def _ask_user_if_they_want_rclone_password(self) -> bool: return input_ == "y" - def _try_set_rclone_password( + def _try_encrypt_rclone_config( self, ) -> None: """Try to encrypt the rclone config file. @@ -982,7 +982,7 @@ def _try_set_rclone_password( If it fails, warn the user the config file is unencrypted. """ try: - self.set_rclone_password() + self.encrypt_rclone_config() except Exception as e: config_path = ( self.cfg.rclone.get_rclone_central_connection_config_filepath() @@ -991,14 +991,17 @@ def _try_set_rclone_password( utils.log_and_raise_error( f"{str(e)}\n" f"Password set up failed.\n" - f"Use set_rclone_password()` to attempt to set the password again (see full error message above).\n" - f"IMPORTANT: The config at {config_path} is not currently encrpyted.\n", + f"Use encrypt_rclone_config()` to attempt to set the password again (see full error message above).\n" + f"IMPORTANT: The config at {config_path} is not currently encrypted.\n", RuntimeError, ) - utils.log_and_message("Password set successfully") + utils.log_and_message( + f"Rclone config file for the central connection " + f"{self.cfg['connection_method']} was successfully encrypted." + ) - def set_rclone_password(self) -> None: + def encrypt_rclone_config(self) -> None: """Encrypt the rclone config file for the central connection.""" if self.cfg.rclone.get_rclone_has_password(): raise RuntimeError( diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 3c39a20f1..fe3dc0c10 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -636,7 +636,7 @@ def setup_aws_connection( def try_setup_rclone_password(self): """""" try: - self.project._try_set_rclone_password() + self.project._try_encrypt_rclone_config() return True, None except BaseException as e: return False, str(e) From 4d7d7ba34dda55507eafe5d3d3f16548674e037c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 13 Oct 2025 15:01:11 +0100 Subject: [PATCH 038/100] Rename from setting rclone password to setting rclone encrpytion. --- datashuttle/configs/rclone_configs.py | 50 +++++++++--------- datashuttle/datashuttle_class.py | 52 ++++++++++--------- datashuttle/tui/interface.py | 4 +- datashuttle/tui/screens/setup_aws.py | 28 +++++----- datashuttle/tui/screens/setup_gdrive.py | 34 ++++-------- datashuttle/tui/screens/setup_ssh.py | 35 +++++++------ datashuttle/utils/rclone.py | 24 ++++----- ...clone_password.py => rclone_encryption.py} | 19 +------ 8 files changed, 113 insertions(+), 133 deletions(-) rename datashuttle/utils/{rclone_password.py => rclone_encryption.py} (94%) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 324ac8366..f985cc4f3 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -23,9 +23,9 @@ class RCloneConfigs: In datashuttle, local filesystem configs uses the Rclone default configuration file, that RClone manages. However, remote transfers to ssh, aws and gdrive are held in separate config files (set using RClone's --config argument). Then being separate - means passwords can be set on these files. + means these files can be separately encrypted. - This class tracks the state on whether a RClone config has a password, as well + This class tracks the state on whether a RClone config is encrypted, as well as provides the default names for the rclone conf (e.g. central__). Parameters @@ -37,15 +37,15 @@ class RCloneConfigs: def __init__(self, datashuttle_configs, config_base_path): self.datashuttle_configs = datashuttle_configs - self.rclone_password_state_file_path = ( + self.rclone_encryption_state_file_path = ( config_base_path / "rclone_ps_state.yaml" ) - def load_rclone_has_password(self): - """Track whether the Rclone config file has a password set. This could be + def load_rclone_config_is_encrypted(self): + """Track whether the Rclone config file is encrypted. This could be read directly from the RClone config file, but requires a subprocess call which can be slow on Windows. As this function is called a lot, we track - this explicitly when a rclone config password is set / removed + this explicitly when a rclone config is encrypted / unencrypted and store to disk between sessions. """ assert self.datashuttle_configs["connection_method"] in [ @@ -54,23 +54,23 @@ def load_rclone_has_password(self): "gdrive", ] - if self.rclone_password_state_file_path.is_file(): - with open(self.rclone_password_state_file_path, "r") as file: - rclone_has_password = yaml.full_load(file) + if self.rclone_encryption_state_file_path.is_file(): + with open(self.rclone_encryption_state_file_path, "r") as file: + rclone_config_is_encrypted = yaml.full_load(file) else: - rclone_has_password = { + rclone_config_is_encrypted = { "ssh": False, "gdrive": False, "aws": False, } - with open(self.rclone_password_state_file_path, "w") as file: - yaml.dump(rclone_has_password, file) + with open(self.rclone_encryption_state_file_path, "w") as file: + yaml.dump(rclone_config_is_encrypted, file) - return rclone_has_password + return rclone_config_is_encrypted - def set_rclone_has_password(self, value): - """Store the current state of the rclone config file password for the `connection_method`. + def set_rclone_config_encryption_state(self, value): + """Store the current state of the rclone config encryption for the `connection_method`. Note that this is stored to disk each call (rather than tracked locally) to ensure it is updated live if updated through the Python API while the TUI is also running. @@ -81,16 +81,16 @@ def set_rclone_has_password(self, value): "gdrive", ] - rclone_has_password = self.load_rclone_has_password() + rclone_config_is_encrypted = self.load_rclone_config_is_encrypted() - rclone_has_password[self.datashuttle_configs["connection_method"]] = ( - value - ) + rclone_config_is_encrypted[ + self.datashuttle_configs["connection_method"] + ] = value - with open(self.rclone_password_state_file_path, "w") as file: - yaml.dump(rclone_has_password, file) + with open(self.rclone_encryption_state_file_path, "w") as file: + yaml.dump(rclone_config_is_encrypted, file) - def get_rclone_has_password( + def get_rclone_config_encryption_state( self, ): """Return whether the config file associated with the current `connection_method`.""" @@ -100,9 +100,9 @@ def get_rclone_has_password( "gdrive", ] - rclone_has_password = self.load_rclone_has_password() + rclone_config_is_encrypted = self.load_rclone_config_is_encrypted() - return rclone_has_password[ + return rclone_config_is_encrypted[ self.datashuttle_configs["connection_method"] ] @@ -151,4 +151,4 @@ def delete_existing_rclone_config_file(self): if rclone_config_filepath.exists(): rclone_config_filepath.unlink() - self.set_rclone_has_password(False) + self.set_rclone_config_encryption_state(False) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 2b692b1bc..2d7f08460 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -45,7 +45,7 @@ gdrive, getters, rclone, - rclone_password, + rclone_encryption, ssh, utils, validation, @@ -841,8 +841,8 @@ def setup_ssh_connection(self) -> None: f"{self.cfg.rclone.get_rclone_central_connection_config_filepath()}.\n" ) - if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_they_want_rclone_password(): + if not self.cfg.rclone.get_rclone_config_encryption_state(): + if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail( @@ -910,8 +910,8 @@ def setup_gdrive_connection(self) -> None: self.cfg, process, log=True ) - if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_they_want_rclone_password(): + if not self.cfg.rclone.get_rclone_config_encryption_state(): + if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -950,8 +950,8 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) - if not self.cfg.rclone.get_rclone_has_password(): - if self._ask_user_if_they_want_rclone_password(): + if not self.cfg.rclone.get_rclone_config_encryption_state(): + if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() rclone.check_successful_connection_and_raise_error_on_fail(self.cfg) @@ -962,14 +962,14 @@ def setup_aws_connection(self) -> None: ds_logger.close_log_filehandler() # ------------------------------------------------------------------------- - # Rclone config password + # Rclone config encryption # ------------------------------------------------------------------------- - def _ask_user_if_they_want_rclone_password(self) -> bool: - """Get user input to determine if they want to set a password on the rclone config.""" + def _ask_user_rclone_encryption(self) -> bool: + """Get user input to determine if they want to encrypt the rclone config.""" input_ = utils.get_user_input( - f"{rclone_password.get_password_explanation_message(self.cfg)}\n" - f"Press 'y' to set password or leave blank to skip." + f"{rclone_encryption.get_explanation_message(self.cfg)}\n" + f"Press 'y' to encrypt the Rclone config or leave blank to skip." ) return input_ == "y" @@ -990,8 +990,9 @@ def _try_encrypt_rclone_config( utils.log_and_raise_error( f"{str(e)}\n" - f"Password set up failed.\n" - f"Use encrypt_rclone_config()` to attempt to set the password again (see full error message above).\n" + f"Config encryption failed.\n" + f"Use encrypt_rclone_config()` to attempt to encrypt the file again " + f"(see full error message above).\n" f"IMPORTANT: The config at {config_path} is not currently encrypted.\n", RuntimeError, ) @@ -1003,27 +1004,28 @@ def _try_encrypt_rclone_config( def encrypt_rclone_config(self) -> None: """Encrypt the rclone config file for the central connection.""" - if self.cfg.rclone.get_rclone_has_password(): + if self.cfg.rclone.get_rclone_config_encryption_state(): raise RuntimeError( - "This config file already has a password set. " - "First, use `remove_rclone_password` to remove it." + "This config file is already encrypted. " + "First, use `remove_rclone_encryption` to remove it." ) - rclone_password.run_rclone_config_encrypt(self.cfg) + rclone_encryption.run_rclone_config_encrypt(self.cfg) - self.cfg.rclone.set_rclone_has_password(True) + self.cfg.rclone.set_rclone_config_encryption_state(True) - def remove_rclone_password(self) -> None: + def remove_rclone_encryption(self) -> None: """Unencrypt the rclone config file for the central connection.""" - if not self.cfg.rclone.get_rclone_has_password(): + if not self.cfg.rclone.get_rclone_config_encryption_state(): raise RuntimeError( - f"The config for the current connection method: {self.cfg['connection_method']} " - f"does not have a password. Cannot remove." + f"The config for the current connection method: " + f"{self.cfg['connection_method']} " + f"is not encrypted. Cannot unencrypt." ) - rclone_password.remove_rclone_password(self.cfg) + rclone_encryption.remove_rclone_encryption(self.cfg) - self.cfg.rclone.set_rclone_has_password(False) + self.cfg.rclone.set_rclone_config_encryption_state(False) # ------------------------------------------------------------------------- # Configs diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index fe3dc0c10..ba8300240 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -630,10 +630,10 @@ def setup_aws_connection( except BaseException as e: return False, str(e) - # Set RClone Password + # Set RClone Encryption # ------------------------------------------------------------------------------------ - def try_setup_rclone_password(self): + def try_setup_rclone_encryption(self): """""" try: self.project._try_encrypt_rclone_config() diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index e39c859c7..828673696 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -11,6 +11,8 @@ from textual.screen import ModalScreen from textual.widgets import Button, Input, Static +from datashuttle.utils import rclone_encryption + class SetupAwsScreen(ModalScreen): """Dialog window that sets up connection to an Amazon Web Service S3 bucket. @@ -61,7 +63,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: """ if event.button.id == "setup_aws_cancel_button": - if self.stage == "ask_password": + if self.stage == "ask_rclone_encryption": message = "AWS Connection Successful!" # self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" @@ -77,8 +79,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: elif self.stage == "use_secret_access_key": self.use_secret_access_key_to_setup_aws_connection() - elif self.stage == "ask_password": - self.set_password() + elif self.stage == "ask_rclone_encryption": + self.set_rclone_encryption() elif self.stage == "finished": self.dismiss() @@ -96,7 +98,7 @@ def prompt_user_for_aws_secret_access_key(self) -> None: def use_secret_access_key_to_setup_aws_connection(self) -> None: """Set up the AWS connection and failure. If success, move onto the - password screen. + rclone_encryption screen. """ secret_access_key = self.query_one( @@ -108,15 +110,13 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: ) if success: - message = ( - f"{rclone_password.get_password_explanation_message(self.cfg)}" - ) + message = f"{rclone_encryption.get_explanation_message(self.cfg)}" self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").remove() self.query_one("#setup_aws_ok_button").label = "Yes" self.query_one("#setup_aws_cancel_button").label = "No" - self.stage = "ask_password" + self.stage = "ask_rclone_encryption" else: message = ( f"AWS setup failed. Please check your configs and secret access key" @@ -129,16 +129,20 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: self.query_one("#setup_aws_ok_button").label = "Retry" self.query_one("#setup_aws_messagebox_message").update(message) - def set_password(self): + def set_rclone_encryption(self): """""" - success, output = self.interface.try_setup_rclone_password() + success, output = self.interface.try_setup_rclone_encryption() if success: - message = "The password was successfully set. Setup complete!" + message = ( + "The rclone_encryption was successfully set. Setup complete!" + ) self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" self.query_one("#setup_aws_cancel_button").remove() self.stage = "finished" else: - message = f"The password set up failed. Exception: {output}" + message = ( + f"The rclone_encryption set up failed. Exception: {output}" + ) self.query_one("#setup_aws_messagebox_message").update(message) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index a6f0938f7..9566360c7 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -22,8 +22,6 @@ Static, ) -from datashuttle.utils import rclone_password - class SetupGdriveScreen(ModalScreen): """Dialog window that sets up a Google Drive connection. @@ -87,9 +85,9 @@ def on_button_pressed(self, event: Button.Pressed) -> None: 4) `setup_gdrive_no_browser_enter_button` : To enter the client secret or config token. - 5) `setup_gdrive_set_password_yes_button` : To set a password on the RClone config file + 5) `setup_gdrive_set_encryption_yes_button` : To set a password on the RClone config file - 6) `setup_gdrive_set_password_no_button` : To skip setting a password on the RClone config file + 6) `setup_gdrive_set_encryption_no_button` : To skip setting a password on the RClone config file 7) `setup_gdrive_finish_button` button : To finish the setup. @@ -148,10 +146,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.gdrive_client_secret, config_token ) - elif event.button.id == "setup_gdrive_set_password_yes_button": - self.set_password() + elif event.button.id == "setup_gdrive_set_encryption_yes_button": + self.set_rclone_encryption() - elif event.button.id == "setup_gdrive_set_password_no_button": + elif event.button.id == "setup_gdrive_set_encryption_no_button": self.set_finish_page("Setup complete!") # Setup the connection (with or without browser) @@ -336,24 +334,12 @@ def setup_gdrive_connection( ) return success, output - # Set password on RClone config + # Set encryption on RClone config # ---------------------------------------------------------------------------------- - def show_password_screen(self): - """""" - message = f"{rclone_password.get_password_explanation_message(self.interface.project.cfg)}" - self.update_message_box_message(message) - - yes_button = Button("Yes", id="setup_gdrive_set_password_yes_button") - no_button = Button("No", id="setup_gdrive_set_password_no_button") - - self.query_one("#setup_gdrive_buttons_horizontal").mount( - yes_button, no_button - ) - - def set_password(self): + def set_rclone_encryption(self): """""" - success, output = self.interface.try_setup_rclone_password() + success, output = self.interface.try_setup_rclone_encryption() if success: self.set_finish_page( @@ -365,8 +351,8 @@ def set_password(self): def set_finish_page(self, message) -> None: """Show the final screen after successful set up.""" - self.query_one("#setup_gdrive_set_password_yes_button").remove() - self.query_one("#setup_gdrive_set_password_no_button").remove() + self.query_one("#setup_gdrive_set_encryption_yes_button").remove() + self.query_one("#setup_gdrive_set_encryption_no_button").remove() self.update_message_box_message(message) self.query_one("#setup_gdrive_buttons_horizontal").mount( diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 368874e84..eb423e0a4 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -8,7 +8,6 @@ from datashuttle.tui.interface import Interface - from textual import work from textual.containers import Container, Horizontal from textual.screen import ModalScreen @@ -18,12 +17,14 @@ Static, ) +from datashuttle.utils import rclone_encryption + class SetupSshScreen(ModalScreen): """Dialog window that sets up an SSH connection. - This asks to confirm the central hostkey, and takes password to setup - SSH key pair as well as setting a password to the RClone config. + This asks to confirm the central hostkey, and takes password to set up + SSH key pair as well as encrypting the RClone config. Due to how textual works, it is simples for each button press to trigger an action (e.g. set up host key) and then set up the widgets @@ -74,7 +75,7 @@ def on_button_pressed(self, event: Button.pressed) -> None: input, multiple attempts are allowed. """ if event.button.id == "setup_ssh_cancel_button": - if self.stage == "set_up_password": + if self.stage == "set_up_encryption": self.show_connection_successful_message() else: self.dismiss() @@ -86,11 +87,11 @@ def on_button_pressed(self, event: Button.pressed) -> None: elif self.stage == "save_hostkeys": self.save_hostkeys_and_prompt_password_input() - elif self.stage == "ask_for_password": + elif self.stage == "ask_for_encryption": self.use_password_to_setup_ssh_key_pairs() - elif self.stage == "set_up_password": - self.try_setup_rclone_password() + elif self.stage == "set_up_encryption": + self.try_setup_rclone_encryption() elif self.stage == "show_success_message": self.show_connection_successful_message() @@ -153,7 +154,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.query_one("#setup_ssh_ok_button").disabled = True self.query_one("#messagebox_message_label").update(message) - self.stage = "ask_for_password" + self.stage = "ask_for_encryption" def use_password_to_setup_ssh_key_pairs(self) -> None: """Set up the SSH key pair using the user-supplied password @@ -172,12 +173,14 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: if success: message = ( f"Connection set up successfully.\n" - f"{rclone_password.get_password_explanation_message(self.cfg)}" + f"{rclone_encryption.get_explanation_message(self.cfg)}" ) self.query_one("#setup_ssh_ok_button").label = "Yes" self.query_one("#setup_ssh_cancel_button").label = "No" self.query_one("#setup_ssh_password_input").visible = False - self.stage = "set_up_password" # Go to password set up screen + self.stage = ( + "set_up_encryption" # Go to rclone encryption set up screen + ) else: message = ( @@ -189,26 +192,26 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.query_one("#messagebox_message_label").update(message) - def try_setup_rclone_password(self): - """Try and set up a password to the RClone config using the system + def try_setup_rclone_encryption(self): + """Try and encrypt the RClone config using the system credential manager. If successful, the next screen confirms success. """ - success, output = self.interface.try_setup_rclone_password() + success, output = self.interface.try_setup_rclone_encryption() if success: - message = "Password successfully set on the config file." + message = "Rclone config file was successfully encrypted." self.query_one("#messagebox_message_label").update(message) self.query_one("#setup_ssh_ok_button").label = "Ok" self.query_one("#setup_ssh_cancel_button").remove() else: - message = f"The password set up failed. Exception: {output}" + message = f"Encryption failed. Exception: {output}" self.query_one("#messagebox_message_label").update(message) self.stage = "show_success_message" @work(exclusive=True, thread=True) def run_interface(self): - self.interface.try_setup_rclone_password() + self.interface.try_setup_rclone_encryption() def show_connection_successful_message(self): """""" diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 6221912d0..6ba824e8b 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -17,7 +17,7 @@ from packaging import version from datashuttle.configs import canonical_configs -from datashuttle.utils import rclone_password, utils +from datashuttle.utils import rclone_encryption, utils def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: @@ -54,7 +54,7 @@ def call_rclone_for_central_connection( cfg, command: str, pipe_std: bool = False ) -> CompletedProcess: """PLACEHOLDER""" - return run_function_that_may_require_central_connection_password( + return run_function_that_requires_encrpyted_rclone_config_access( cfg, lambda: call_rclone(command, pipe_std) ) @@ -104,7 +104,7 @@ def call_rclone_through_script_for_central_connection( shell=False, ) - output = run_function_that_may_require_central_connection_password( + output = run_function_that_requires_encrpyted_rclone_config_access( cfg, lambda_func ) @@ -147,7 +147,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( """ lambda_func = lambda: process.communicate() - stdout, stderr = run_function_that_may_require_central_connection_password( + stdout, stderr = run_function_that_requires_encrpyted_rclone_config_access( cfg, lambda_func ) @@ -160,7 +160,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( return stdout, stderr -def run_function_that_may_require_central_connection_password( +def run_function_that_requires_encrpyted_rclone_config_access( cfg, lambda_func ): """ """ @@ -182,15 +182,15 @@ def run_function_that_may_require_central_connection_password( f"Please set up the {cfg['connection_method']} connection again." ) - set_password = cfg.rclone.get_rclone_has_password() + is_encrypted = cfg.rclone.get_rclone_config_encryption_state() - if set_password: - rclone_password.set_credentials_as_password_command(cfg) + if is_encrypted: + rclone_encryption.set_credentials_as_password_command(cfg) results = lambda_func() - if set_password: - rclone_password.remove_credentials_as_password_command() + if is_encrypted: + rclone_encryption.remove_credentials_as_password_command() return results @@ -585,8 +585,8 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.rclone.get_rclone_has_password(): - rclone_password.remove_credentials_as_password_command() + if cfg.rclone.get_rclone_config_encryption_state(): + rclone_encryption.remove_credentials_as_password_command() return output diff --git a/datashuttle/utils/rclone_password.py b/datashuttle/utils/rclone_encryption.py similarity index 94% rename from datashuttle/utils/rclone_password.py rename to datashuttle/utils/rclone_encryption.py index 0d381717a..b26b5338b 100644 --- a/datashuttle/utils/rclone_password.py +++ b/datashuttle/utils/rclone_encryption.py @@ -191,7 +191,7 @@ def run_rclone_config_encrypt(cfg: Configs): remove_credentials_as_password_command() -def remove_rclone_password(cfg): +def remove_rclone_encryption(cfg): """""" set_credentials_as_password_command(cfg) @@ -242,22 +242,7 @@ def get_password_filepath( return base_path / f"{cfg.rclone.get_rclone_config_name()}.xml" -def run_raise_if_fail(command, command_description): - output = run_subprocess.run( - command, - shell=True, # TODO: handle shell - capture_output=True, - text=True, - ) - - if output.returncode != 0: - raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}\n" - f"\n--- STDERR ---\n{output.stderr}\n" - ) - - -def get_password_explanation_message( +def get_explanation_message( cfg: Configs, ): # TODO: type when other PR is merged """""" From a2246c20dcf8e56792ce017daed68590a6975ee8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 13 Oct 2025 16:58:36 +0100 Subject: [PATCH 039/100] Begin adding docstrings and type hints. --- datashuttle/utils/rclone_encryption.py | 129 ++++++++++++++++++++----- 1 file changed, 103 insertions(+), 26 deletions(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index b26b5338b..1afb55a6e 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -1,3 +1,7 @@ +"""Module for encrypthing the RClone config file. Methods based on: +https://rclone.org/docs/#configuration-encryption +""" + from __future__ import annotations from typing import TYPE_CHECKING @@ -9,13 +13,14 @@ import platform import shutil import subprocess +from pathlib import Path from datashuttle.configs import canonical_folders from datashuttle.utils import utils -def save_credentials_password(cfg): - """""" +def save_credentials_password(cfg: Configs) -> None: + """Use the system password manager to set up a password for the Rclone config file encryption.""" if platform.system() == "Windows": set_password_windows(cfg) elif platform.system() == "Linux": @@ -24,9 +29,16 @@ def save_credentials_password(cfg): set_password_macos(cfg) -def set_password_windows(cfg: Configs): - """""" - password_filepath = get_password_filepath(cfg) +def set_password_windows(cfg: Configs) -> None: + """Generate and securely store a random password in a Windows Credential XML file. + + Use PowerShell to create a random password associated with the name 'rclone'. + The password is stored as a PowerShell `PSCredential` object that can only + be decrypted by the same Windows user account that created it. + + This password is later used to encrypt the Rclone config file. + """ + password_filepath = get_windows_password_filepath(cfg) if password_filepath.exists(): password_filepath.unlink() @@ -57,8 +69,16 @@ def set_password_windows(cfg: Configs): ) -def set_password_linux(cfg): - """""" +def set_password_linux(cfg: Configs) -> None: + """Generate and securely store a random password using the Linux `pass` utility. + + This function generates a random password and stores it in the user's + GPG-encrypted password store via the `pass` command-line tool. + + The `pass` utility must be installed and initialized with a GPG ID on the + current user account (via `pass init `). If it is not initialized, + a RuntimeError will be raised. + """ output = subprocess.run( "pass --help", shell=True, @@ -104,8 +124,15 @@ def set_password_linux(cfg): ) -def set_password_macos(cfg: Configs): - """""" +def set_password_macos(cfg: Configs) -> None: + """Generate and store a password using the macOS Keychain. + + This function generates a random password and stores it in the macOS Keychain + using the built-in `security` command-line tool. + + The password is generated using OpenSSL with 40 random base64 characters and + is securely saved to the user's login Keychain. + """ output = subprocess.run( f"security add-generic-password -a datashuttle -s {cfg.rclone.get_rclone_config_name()} -w $(openssl rand -base64 40) -U", shell=True, @@ -121,10 +148,22 @@ def set_password_macos(cfg: Configs): ) -def set_credentials_as_password_command(cfg): - """""" +def set_credentials_as_password_command(cfg: Configs) -> None: + """Configure the RClone password retrieval command based on the operating system. + + This function sets the `RCLONE_PASSWORD_COMMAND` environment variable so that + RClone can securely retrieve stored credentials + + - Windows : Uses PowerShell to decrypt a previously exported `PSCredential` + object from the `.clixml` file created by `set_password_windows()`. + - Linux : Uses the `pass` command-line utility to fetch the stored password + from the user's GPG-encrypted password store. + - macOS : Uses the built-in `security` tool to read the password + from the user's Keychain, associated with the account name `datashuttle` and + the rclone service name. + """ if platform.system() == "Windows": - password_filepath = get_password_filepath(cfg) + password_filepath = get_windows_password_filepath(cfg) assert password_filepath.exists(), ( "Critical error: password file not found when setting password command." @@ -135,8 +174,6 @@ def set_credentials_as_password_command(cfg): raise RuntimeError("powershell.exe not found in PATH") # Escape single quotes inside PowerShell string by doubling them - # safe_path = str(filepath).replace("'", "''") - cmd = ( f'{shell} -NoProfile -Command "Write-Output (' f"[System.Runtime.InteropServices.Marshal]::PtrToStringAuto(" @@ -157,8 +194,19 @@ def set_credentials_as_password_command(cfg): ) -def run_rclone_config_encrypt(cfg: Configs): - """""" +def run_rclone_config_encrypt(cfg: Configs) -> None: + """Encrypt the rclone config file using an OS-native secret. + + This function: + 1) Generates/stores a random password using the platform-specific backend + (Windows PSCredential, Linux `pass`, or macOS Keychain) via + `save_credentials_password(cfg)`. + 2) Sets `RCLONE_PASSWORD_COMMAND` so rclone can retrieve the secret on demand + via `set_credentials_as_password_command(cfg)`. + 3) Runs `rclone config encryption set --config ` to encrypt the config. + 4) Cleans up by removing the password command environment variable with + `remove_credentials_as_password_command()`. + """ rclone_config_path = ( cfg.rclone.get_rclone_central_connection_config_filepath() ) @@ -191,8 +239,13 @@ def run_rclone_config_encrypt(cfg: Configs): remove_credentials_as_password_command() -def remove_rclone_encryption(cfg): - """""" +def remove_rclone_encryption(cfg: Configs) -> None: + """Remove encryption from an Rclone config file. + + Set the credentials one last time to remove encryption from + the RClone config file. Once removed, clean up the password + as stored with the system credential manager. + """ set_credentials_as_password_command(cfg) config_filepath = ( @@ -215,7 +268,28 @@ def remove_rclone_encryption(cfg): remove_credentials_as_password_command() if platform.system() == "Windows": - get_password_filepath(cfg).unlink() + get_windows_password_filepath(cfg).unlink() + + elif platform.system() == "Linux": + name = cfg.rclone.get_rclone_config_name() + subprocess.run( + ["pass", "rm", "-f", name], + check=False, + ) + + elif platform.system() == "Darwin": + service = cfg.rclone.get_rclone_config_name() + subprocess.run( + [ + "security", + "delete-generic-password", + "-a", + "datashuttle", + "-s", + service, + ], + check=False, + ) utils.log_and_message( f"Password removed from rclone config file: {config_filepath}" @@ -227,10 +301,10 @@ def remove_credentials_as_password_command(): os.environ.pop("RCLONE_PASSWORD_COMMAND") -def get_password_filepath( - cfg, -): # Configs # TODO: datashuttle_path should be on configs? - """""" +def get_windows_password_filepath( + cfg: Configs, +) -> Path: + """Get the canonical location where datashuttle stores the windows credentials.""" assert cfg["connection_method"] in ["aws", "gdrive", "ssh"], ( "password should only be set for ssh, aws, gdrive." ) @@ -244,10 +318,13 @@ def get_password_filepath( def get_explanation_message( cfg: Configs, -): # TODO: type when other PR is merged - """""" +) -> str: + """Explaining rclone's default credential storage and OS-specific encryption options. + + Displayed in both the Python API and the TUI. + """ system_pass_manager = { - "Windows": "Windows Credential Manager", + "Windows": "PSCredential", "Linux": "the `pass` program", "Darwin": "macOS built-in `security` tool", } From c7d3fb11f2fd2639bc395c527e5f89993795583f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 13 Oct 2025 17:32:28 +0100 Subject: [PATCH 040/100] Add docstrings and type hints. --- datashuttle/configs/rclone_configs.py | 8 ++--- datashuttle/datashuttle_class.py | 3 +- datashuttle/tui/interface.py | 2 +- datashuttle/tui/screens/setup_aws.py | 2 +- datashuttle/tui/screens/setup_gdrive.py | 2 +- datashuttle/tui/screens/setup_ssh.py | 10 +++--- datashuttle/utils/folders.py | 2 +- datashuttle/utils/rclone.py | 43 ++++++++++++++++--------- 8 files changed, 42 insertions(+), 30 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index f985cc4f3..427517917 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -41,7 +41,7 @@ def __init__(self, datashuttle_configs, config_base_path): config_base_path / "rclone_ps_state.yaml" ) - def load_rclone_config_is_encrypted(self): + def load_rclone_config_is_encrypted(self) -> dict: """Track whether the Rclone config file is encrypted. This could be read directly from the RClone config file, but requires a subprocess call which can be slow on Windows. As this function is called a lot, we track @@ -69,7 +69,7 @@ def load_rclone_config_is_encrypted(self): return rclone_config_is_encrypted - def set_rclone_config_encryption_state(self, value): + def set_rclone_config_encryption_state(self, value: bool) -> None: """Store the current state of the rclone config encryption for the `connection_method`. Note that this is stored to disk each call (rather than tracked locally) to ensure @@ -92,7 +92,7 @@ def set_rclone_config_encryption_state(self, value): def get_rclone_config_encryption_state( self, - ): + ) -> dict: """Return whether the config file associated with the current `connection_method`.""" assert self.datashuttle_configs["connection_method"] in [ "ssh", @@ -143,7 +143,7 @@ def make_rclone_transfer_options( "dry_run": dry_run, } - def delete_existing_rclone_config_file(self): + def delete_existing_rclone_config_file(self) -> None: """ """ rclone_config_filepath = ( self.get_rclone_central_connection_config_filepath() diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 2d7f08460..751c7a5e9 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -979,7 +979,7 @@ def _try_encrypt_rclone_config( ) -> None: """Try to encrypt the rclone config file. - If it fails, warn the user the config file is unencrypted. + If it fails, error and let the user the config file is unencrypted. """ try: self.encrypt_rclone_config() @@ -1224,6 +1224,7 @@ def get_config_path(self) -> Path: @check_configs_set def get_rclone_central_config_path(self) -> Path: + """Get the path to the Rclone config for the current `connection_method`.""" return rclone.get_rclone_config_filepath(self.cfg) @check_configs_set diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index ba8300240..c48ffca26 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -634,7 +634,7 @@ def setup_aws_connection( # ------------------------------------------------------------------------------------ def try_setup_rclone_encryption(self): - """""" + """Try and encrypt the RClone config file for the current `connection_method`.""" try: self.project._try_encrypt_rclone_config() return True, None diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 828673696..4b3add182 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -130,7 +130,7 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: self.query_one("#setup_aws_messagebox_message").update(message) def set_rclone_encryption(self): - """""" + """Try and encrypt the Rclone config file and inform the user of success / failure.""" success, output = self.interface.try_setup_rclone_encryption() if success: diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 9566360c7..6300c9c2d 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -338,7 +338,7 @@ def setup_gdrive_connection( # ---------------------------------------------------------------------------------- def set_rclone_encryption(self): - """""" + """Try and encrypt the Rclone config file and inform the user of success / failure.""" success, output = self.interface.try_setup_rclone_encryption() if success: diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index eb423e0a4..d142f25fb 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -8,7 +8,6 @@ from datashuttle.tui.interface import Interface -from textual import work from textual.containers import Container, Horizontal from textual.screen import ModalScreen from textual.widgets import ( @@ -209,13 +208,12 @@ def try_setup_rclone_encryption(self): self.stage = "show_success_message" - @work(exclusive=True, thread=True) - def run_interface(self): - self.interface.try_setup_rclone_encryption() - def show_connection_successful_message(self): - """""" + """Show the final screen indicating the connection was successfully set up.""" self.query_one("#setup_ssh_ok_button").label = "Finish" + + # Depending on what was the previous screen, `setup_ssh_cancel_button` + # may or may not be displayed. try: self.query_one("#setup_ssh_cancel_button").remove() except BaseException: diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 7f2e2d11a..dcb903919 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -697,7 +697,7 @@ def search_central_via_connection( """ rclone_config_name = cfg.rclone.get_rclone_config_name( cfg["connection_method"] - ) # TODO: this is not good because we get the config name here and in get_config_arg + ) output = rclone.call_rclone_for_central_connection( cfg, diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 6ba824e8b..002b2313c 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Literal, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional if TYPE_CHECKING: from datashuttle.configs.config_class import Configs @@ -53,8 +53,14 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: def call_rclone_for_central_connection( cfg, command: str, pipe_std: bool = False ) -> CompletedProcess: - """PLACEHOLDER""" - return run_function_that_requires_encrpyted_rclone_config_access( + """Call RClone when the config file may need to be unencrypted. + + This is a convenience function to call RClone in places where + the config file may need to be unencrypted. This is for connecting + to the central storage through aws, ssh or gdrive. It wraps the + function call in a set-up / teardown of the config password. + """ + return run_function_that_requires_encrypted_rclone_config_access( cfg, lambda: call_rclone(command, pipe_std) ) @@ -104,7 +110,7 @@ def call_rclone_through_script_for_central_connection( shell=False, ) - output = run_function_that_requires_encrpyted_rclone_config_access( + output = run_function_that_requires_encrypted_rclone_config_access( cfg, lambda_func ) @@ -147,7 +153,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( """ lambda_func = lambda: process.communicate() - stdout, stderr = run_function_that_requires_encrpyted_rclone_config_access( + stdout, stderr = run_function_that_requires_encrypted_rclone_config_access( cfg, lambda_func ) @@ -160,10 +166,15 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( return stdout, stderr -def run_function_that_requires_encrpyted_rclone_config_access( +def run_function_that_requires_encrypted_rclone_config_access( cfg, lambda_func -): - """ """ +) -> Any: + """Run command that requires possibly encrypted Rclone config file. + + The Rclone config file may be encrypted for aws, gdrive or ssh connections. + In this case we need to set an environment variable to tell Rclone how + to decrypt the config file (and remove the variable afterwards). + """ from datashuttle import get_datashuttle_version # avoid circular import rclone_config_filepath = ( @@ -178,7 +189,8 @@ def run_function_that_requires_encrpyted_rclone_config_access( ) else: raise RuntimeError( - f"An unexpected error occurred. Could not find the rclone config file at: {rclone_config_filepath}\n" + f"An unexpected error occurred. Could not find the rclone config " + f"file at: {rclone_config_filepath}\n" f"Please set up the {cfg['connection_method']} connection again." ) @@ -187,10 +199,11 @@ def run_function_that_requires_encrpyted_rclone_config_access( if is_encrypted: rclone_encryption.set_credentials_as_password_command(cfg) - results = lambda_func() - - if is_encrypted: - rclone_encryption.remove_credentials_as_password_command() + try: + results = lambda_func() + finally: + if is_encrypted: + rclone_encryption.remove_credentials_as_password_command() return results @@ -419,7 +432,7 @@ def delete_existing_rclone_config_file(cfg: Configs): def get_config_arg(cfg: Configs) -> str: - """TODO PLACEHOLDER.""" + """Get the full argument to run Rclone commands with a specific config.""" rclone_config_path = ( cfg.rclone.get_rclone_central_connection_config_filepath() ) @@ -470,7 +483,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: def get_rclone_config_filepath(cfg: Configs) -> Path: - """""" + """Get the path to the central Rclone config for the current `connection_method`.""" if cfg["connection_method"] in ["aws", "ssh", "gdrive"]: config_filepath = ( cfg.rclone.get_rclone_central_connection_config_filepath() From 54cd2ee81d4b3a4883716d9307d06253f4882041 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 15:56:12 +0000 Subject: [PATCH 041/100] Fix linting. --- datashuttle/configs/canonical_folders.py | 5 +- datashuttle/configs/rclone_configs.py | 27 ++++--- datashuttle/tui/screens/setup_aws.py | 76 +++++-------------- datashuttle/tui/screens/setup_gdrive.py | 3 +- datashuttle/tui/screens/setup_ssh.py | 8 +- datashuttle/tui/tabs/transfer.py | 3 +- datashuttle/utils/rclone.py | 13 +++- datashuttle/utils/rclone_encryption.py | 9 ++- .../pages/get_started/set-up-a-project.md | 2 +- 9 files changed, 58 insertions(+), 88 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index b2cb180a6..d9426bcb7 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -96,8 +96,9 @@ def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: def get_rclone_config_base_path(): - """Get the path to the Rclone config file. This is used for - RClone config files for transfer targets (ssh, aws, gdrive). + """Return the path to the Rclone config file. + + This is used for RClone config files for transfer targets (ssh, aws, gdrive). This should match where RClone itself stores the config by default, as described here: https://rclone.org/docs/#config-string diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 427517917..151681d57 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -3,12 +3,12 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: + from pathlib import Path + from datashuttle.utils.custom_types import ( OverwriteExistingFiles, ) -from pathlib import Path - import yaml from datashuttle.configs import canonical_folders @@ -16,9 +16,10 @@ class RCloneConfigs: - """This class manages the RClone configuration file. This is a file that RClone creates - to hold all information about local and remote transfer targets. For example, the - ssh RClone config holds the private key. + """Class to manage the RClone configuration file. + + This is a file that RClone creates to hold all information about local and + remote transfer targets. For example, the ssh RClone config holds the private key. In datashuttle, local filesystem configs uses the Rclone default configuration file, that RClone manages. However, remote transfers to ssh, aws and gdrive are held in @@ -36,17 +37,19 @@ class RCloneConfigs: """ def __init__(self, datashuttle_configs, config_base_path): + """Construct the class.""" self.datashuttle_configs = datashuttle_configs self.rclone_encryption_state_file_path = ( config_base_path / "rclone_ps_state.yaml" ) def load_rclone_config_is_encrypted(self) -> dict: - """Track whether the Rclone config file is encrypted. This could be - read directly from the RClone config file, but requires a subprocess call - which can be slow on Windows. As this function is called a lot, we track - this explicitly when a rclone config is encrypted / unencrypted - and store to disk between sessions. + """Track whether the Rclone config file is encrypted. + + This could be read directly from the RClone config file, but requires + a subprocess call which can be slow on Windows. As this function is + called a lot, we track this explicitly when a rclone config is + encrypted / unencrypted and store to disk between sessions. """ assert self.datashuttle_configs["connection_method"] in [ "ssh", @@ -116,7 +119,7 @@ def get_rclone_config_name( return f"central_{self.datashuttle_configs.project_name}_{connection_method}" def get_rclone_central_connection_config_filepath(self) -> Path: - """The full filepath to the rclone `.conf` config file""" + """Return the full filepath to the rclone `.conf` config file.""" return ( canonical_folders.get_rclone_config_base_path() / f"{self.get_rclone_config_name()}.conf" @@ -144,7 +147,7 @@ def make_rclone_transfer_options( } def delete_existing_rclone_config_file(self) -> None: - """ """ + """Delete the Rclone config file if it exists.""" rclone_config_filepath = ( self.get_rclone_central_connection_config_filepath() ) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 4b3add182..8527e1dd9 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -11,8 +11,6 @@ from textual.screen import ModalScreen from textual.widgets import Button, Input, Static -from datashuttle.utils import rclone_encryption - class SetupAwsScreen(ModalScreen): """Dialog window that sets up connection to an Amazon Web Service S3 bucket. @@ -28,7 +26,7 @@ def __init__(self, interface: Interface) -> None: super(SetupAwsScreen, self).__init__() self.interface = interface - self.stage = "init" + self.stage = 0 def compose(self) -> ComposeResult: """Set widgets on the SetupAwsScreen.""" @@ -54,35 +52,18 @@ def on_mount(self) -> None: self.query_one("#setup_aws_secret_access_key_input").visible = False def on_button_pressed(self, event: Button.Pressed) -> None: - """Handle button press on the screen. - - The `setup_aws_ok_button` is used for all 'positive' events ('Yes, Ok') - and 'setup_aws_cancel_button' is used for 'negative' events ('No', 'Cancel'). - The appropriate action to take on the button press is determined by the - current stage. - - """ + """Handle button press on the screen.""" if event.button.id == "setup_aws_cancel_button": - if self.stage == "ask_rclone_encryption": - message = "AWS Connection Successful!" # - self.query_one("#setup_aws_messagebox_message").update(message) - self.query_one("#setup_aws_ok_button").label = "Finish" - self.query_one("#setup_aws_cancel_button").remove() - self.stage = "finished" - else: - self.dismiss() + self.dismiss() - elif event.button.id == "setup_aws_ok_button": - if self.stage == "init": + if event.button.id == "setup_aws_ok_button": + if self.stage == 0: self.prompt_user_for_aws_secret_access_key() - elif self.stage == "use_secret_access_key": + elif self.stage == 1: self.use_secret_access_key_to_setup_aws_connection() - elif self.stage == "ask_rclone_encryption": - self.set_rclone_encryption() - - elif self.stage == "finished": + elif self.stage == 2: self.dismiss() def prompt_user_for_aws_secret_access_key(self) -> None: @@ -92,15 +73,10 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").visible = True - self.query_one("#setup_aws_ok_button") - - self.stage = "use_secret_access_key" + self.stage += 1 def use_secret_access_key_to_setup_aws_connection(self) -> None: - """Set up the AWS connection and failure. If success, move onto the - rclone_encryption screen. - - """ + """Set up the AWS connection and inform user of success or failure.""" secret_access_key = self.query_one( "#setup_aws_secret_access_key_input" ).value @@ -110,13 +86,11 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: ) if success: - message = f"{rclone_encryption.get_explanation_message(self.cfg)}" - self.query_one("#setup_aws_messagebox_message").update(message) + message = "AWS Connection Successful!" + self.query_one( + "#setup_aws_secret_access_key_input" + ).visible = False - self.query_one("#setup_aws_secret_access_key_input").remove() - self.query_one("#setup_aws_ok_button").label = "Yes" - self.query_one("#setup_aws_cancel_button").label = "No" - self.stage = "ask_rclone_encryption" else: message = ( f"AWS setup failed. Please check your configs and secret access key" @@ -126,23 +100,7 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: "#setup_aws_secret_access_key_input" ).disabled = True - self.query_one("#setup_aws_ok_button").label = "Retry" - self.query_one("#setup_aws_messagebox_message").update(message) - - def set_rclone_encryption(self): - """Try and encrypt the Rclone config file and inform the user of success / failure.""" - success, output = self.interface.try_setup_rclone_encryption() - - if success: - message = ( - "The rclone_encryption was successfully set. Setup complete!" - ) - self.query_one("#setup_aws_messagebox_message").update(message) - self.query_one("#setup_aws_ok_button").label = "Finish" - self.query_one("#setup_aws_cancel_button").remove() - self.stage = "finished" - else: - message = ( - f"The rclone_encryption set up failed. Exception: {output}" - ) - self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_ok_button").label = "Finish" + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_cancel_button").disabled = True + self.stage += 1 diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 6300c9c2d..910066c6e 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -13,9 +13,8 @@ from textual import work from textual.containers import Container, Horizontal, Vertical -from textual.dom import NoMatches -from textual.screen import ModalScreen from textual.css.query import NoMatches +from textual.screen import ModalScreen from textual.widgets import ( Button, Input, diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index d142f25fb..3073cc3c3 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -156,8 +156,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage = "ask_for_encryption" def use_password_to_setup_ssh_key_pairs(self) -> None: - """Set up the SSH key pair using the user-supplied password - to the central server. + """Set up the SSH key pair using the user-supplied password to the central server. Next, set up the request asking if they would like to set a (separate) password on their RClone config, using the @@ -192,8 +191,9 @@ def use_password_to_setup_ssh_key_pairs(self) -> None: self.query_one("#messagebox_message_label").update(message) def try_setup_rclone_encryption(self): - """Try and encrypt the RClone config using the system - credential manager. If successful, the next screen confirms success. + """Try and encrypt the RClone config using the system credential manager. + + If successful, the next screen confirms success. """ success, output = self.interface.try_setup_rclone_encryption() diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index a6362ad85..07d37c734 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -16,6 +16,7 @@ from rich.text import Text from textual import work from textual.containers import Container, Horizontal, Vertical +from textual.css.query import NoMatches from textual.widgets import ( Button, Checkbox, @@ -217,8 +218,6 @@ def on_mount(self) -> None: "#transfer_tab_overwrite_select", "#transfer_tab_dry_run_checkbox", ]: - from textual.css.query import NoMatches - try: # if checkbox is removed by user, hard to predict, skip. self.query_one(id).tooltip = get_tooltip(id) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 002b2313c..bce9b14f6 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -3,6 +3,9 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional if TYPE_CHECKING: + from pathlib import Path + from subprocess import CompletedProcess + from datashuttle.configs.config_class import Configs from datashuttle.utils.custom_types import TopLevelFolder @@ -11,8 +14,6 @@ import shlex import subprocess import tempfile -from pathlib import Path -from subprocess import CompletedProcess from packaging import version @@ -66,7 +67,7 @@ def call_rclone_for_central_connection( def call_rclone_through_script_for_central_connection( - cfg, command: str + cfg: Configs, command: str ) -> CompletedProcess: """Call rclone through a script. @@ -75,6 +76,9 @@ def call_rclone_through_script_for_central_connection( Parameters ---------- + cfg + Datashuttle Configs class. + ---------- command Full command to run with RClone. @@ -234,6 +238,9 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- + cfg + datashuttle Configs class + rclone_config_name canonical config name, generated by datashuttle.cfg.rclone.get_rclone_config_name() diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 1afb55a6e..c822551ae 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -1,5 +1,6 @@ -"""Module for encrypthing the RClone config file. Methods based on: -https://rclone.org/docs/#configuration-encryption +"""Module for encrypting the RClone config file. + +Methods based on: https://rclone.org/docs/#configuration-encryption. """ from __future__ import annotations @@ -7,13 +8,14 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + from pathlib import Path + from datashuttle.configs.configs_class import Configs import os import platform import shutil import subprocess -from pathlib import Path from datashuttle.configs import canonical_folders from datashuttle.utils import utils @@ -297,6 +299,7 @@ def remove_rclone_encryption(cfg: Configs) -> None: def remove_credentials_as_password_command(): + """Tidy up the rclone password environment variable.""" if "RCLONE_PASSWORD_COMMAND" in os.environ: os.environ.pop("RCLONE_PASSWORD_COMMAND") diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index cda422c2a..62e121689 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -573,7 +573,7 @@ This means the file is only uncryptable on your local machine or user) CHECK USE TODO: think more about the credentials file... its' stupid to have this itself plain text in datashuttle? -Despite this layer of security, it is not reccomended to use datashuttle for remote connectivity on +Despite this layer of security, it is not recommended to use datashuttle for remote connectivity on a machine to which you do not have secure access, even with password protection of the RClone config. TODO: test if `pass` is not installed on linux that the error is propagated to the TUI properly From eaa06b6e02a684f0c6609af1d195370ef156e4d5 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 16:28:23 +0000 Subject: [PATCH 042/100] Add first rclone encryption test. --- .../test_rclone_encryption.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/tests_integration/test_rclone_encryption.py diff --git a/tests/tests_integration/test_rclone_encryption.py b/tests/tests_integration/test_rclone_encryption.py new file mode 100644 index 000000000..cda3f6e24 --- /dev/null +++ b/tests/tests_integration/test_rclone_encryption.py @@ -0,0 +1,51 @@ +from datashuttle.utils import rclone_encryption + +from ..base import BaseTest +from ..tests_transfers.ssh import ssh_test_utils + + +class TestRcloneEncryption(BaseTest): + def test_set_and_remove_password(self, project): + """""" + ssh_test_utils.setup_project_for_ssh( + project, + ) + + rclone_config_path = ( + project.cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + if rclone_config_path.exists(): + rclone_config_path.unlink() + + import textwrap + + config_content = textwrap.dedent(f"""\ + [{project.cfg.rclone.get_rclone_config_name()}] + type = sftp + host = ssh.swc.ucl.ac.uk + user = jziminski + port = 22 + key_file = C:/Users/Jzimi/.datashuttle/my_project_name/my_project_name_ssh_key + shell_type = unix + md5sum_command = md5sum + sha1sum_command = sha1sum + """) + + # Write to file + with open(rclone_config_path, "w") as file: + file.write(config_content) + + rclone_encryption.run_rclone_config_encrypt(project.cfg) + + with open(rclone_config_path, "r", encoding="utf-8") as f: + first_line = f.readline().strip() + + assert first_line == "# Encrypted rclone configuration File" + + rclone_encryption.remove_rclone_encryption(project.cfg) + + with open(rclone_config_path, "r", encoding="utf-8") as f: + first_line = f.readline().strip() + + assert first_line == f"[{project.cfg.rclone.get_rclone_config_name()}]" From 3b4ac9d550f62b7ce2968349e0e759b5b7940594 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 16:40:54 +0000 Subject: [PATCH 043/100] Update yaml for testing. --- .github/workflows/code_test_and_deploy.yml | 26 +--------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index e6e8fcb14..85e4011dc 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -69,33 +69,9 @@ jobs: # run SSH tests only on Linux because Windows and macOS # are already run within a virtual container and so cannot # run Linux containers because nested containerisation is disabled. - - name: Test SSH (Linux only) - if: runner.os == 'Linux' - run: | - sudo service mysql stop # free up port 3306 for ssh tests - pytest tests/tests_transfers/ssh - - - name: Test Google Drive - env: - GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} - GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} - GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} - GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} - run: | - pytest tests/tests_transfers/gdrive - - - name: Test AWS - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} - run: | - pytest tests/tests_transfers/aws - - name: All Other Tests run: | - pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws + pytest -k test_rclone_encryption build_sdist_wheels: From f9d3dbf8bad3260c339410ff708404a9fe4e34ea Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 16:50:05 +0000 Subject: [PATCH 044/100] install pass on linux. --- .github/workflows/code_test_and_deploy.yml | 36 +++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 85e4011dc..eaac10af0 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -69,7 +69,41 @@ jobs: # run SSH tests only on Linux because Windows and macOS # are already run within a virtual container and so cannot # run Linux containers because nested containerisation is disabled. - - name: All Other Tests +# - name: Test SSH (Linux only) +# if: runner.os == 'Linux' +# run: | +# sudo service mysql stop # free up port 3306 for ssh tests +# pytest tests/tests_transfers/ssh +# +# - name: Test Google Drive +# env: +# GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} +# GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} +# GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} +# GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} +# run: | +# pytest tests/tests_transfers/gdrive + +# - name: Test AWS +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# AWS_REGION: ${{ secrets.AWS_REGION }} +# AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} +# run: | +# pytest tests/tests_transfers/aws + +# - name: All Other Tests +# run: | +# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws + + - name: Install pass on Linux + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y pass + + - name: RClone Encryption run: | pytest -k test_rclone_encryption From f54d8631ce13baf3d88f1524fc0c7d88510eccf4 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 17:25:00 +0000 Subject: [PATCH 045/100] Initialise pass on linux. --- .github/workflows/code_test_and_deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index eaac10af0..f9b8417f2 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -100,8 +100,10 @@ jobs: - name: Install pass on Linux if: runner.os == 'Linux' run: | - sudo apt-get update - sudo apt-get install -y pass + sudo apt-get update && sudo apt-get install -y pass gnupg + export GNUPGHOME=$(mktemp -d) + gpg --batch --passphrase '' --quick-gen-key "CI Key" default default never + pass init "CI Key" - name: RClone Encryption run: | From c1d1c6552cbc18e109c30cb852057eb701c46dba Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 30 Oct 2025 18:03:22 +0000 Subject: [PATCH 046/100] Try a different pass set up command. --- .github/workflows/code_test_and_deploy.yml | 24 ++++++++++++++++++---- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index f9b8417f2..05dc24277 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -100,13 +100,29 @@ jobs: - name: Install pass on Linux if: runner.os == 'Linux' run: | - sudo apt-get update && sudo apt-get install -y pass gnupg - export GNUPGHOME=$(mktemp -d) - gpg --batch --passphrase '' --quick-gen-key "CI Key" default default never - pass init "CI Key" + set -euo pipefail + sudo apt-get update + sudo apt-get install -y pass gnupg git + + # Create a dedicated GPG home for this job + export GNUPGHOME="$(mktemp -d)" + echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV" # <-- make it available to later steps + + # Generate a non-interactive key (no passphrase), no expiry + gpg --batch --yes --pinentry-mode loopback --passphrase '' \ + --quick-gen-key "CI Key " default default 0 + + # Initialize pass with the key fingerprint (more robust than UID) + FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')" + pass init "$FPR" + + # (Optional) smoke test: ensure pass can encrypt + printf '%s\n' "$(openssl rand -base64 16)" | pass insert -m -f ci/smoke-test - name: RClone Encryption run: | + set -euo pipefail + # GNUPGHOME is available here because we wrote it to $GITHUB_ENV pytest -k test_rclone_encryption diff --git a/pyproject.toml b/pyproject.toml index 2e9402411..415a56cae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,6 +164,6 @@ ignore = [ datashuttle = "datashuttle.tui_launcher:main" [tool.codespell] -skip = '.git,*.pdf,*.svg' +skip = '.git,*.pdf,*.svg,*.yml' # # ignore-words-list = '' From 6d1dcbd159173f71a93c55a053afad231d91faa6 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 31 Oct 2025 17:56:54 +0000 Subject: [PATCH 047/100] Fix ssh and gdrive tests. --- .github/workflows/code_test_and_deploy.yml | 68 ++++++++-------- datashuttle/tui/interface.py | 14 ++-- datashuttle/tui/screens/setup_gdrive.py | 15 ++++ datashuttle/utils/gdrive.py | 68 +--------------- datashuttle/utils/rclone.py | 72 ++++++++++++++++- datashuttle/utils/rclone_encryption.py | 4 +- tests/test_utils.py | 35 +-------- .../test_rclone_encryption.py | 8 +- tests/tests_transfers/base_transfer.py | 35 ++++++++- .../gdrive/gdrive_test_utils.py | 20 +++-- .../gdrive/test_gdrive_transfer.py | 5 +- .../gdrive/test_tui_setup_gdrive.py | 78 ++++++++++++++++--- tests/tests_transfers/ssh/ssh_test_utils.py | 2 +- .../test_gdrive_preliminary_setup.py | 6 +- 14 files changed, 255 insertions(+), 175 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 05dc24277..e53d47e6d 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -66,38 +66,8 @@ jobs: python -m pip install --upgrade pip pip install .[dev] - # run SSH tests only on Linux because Windows and macOS - # are already run within a virtual container and so cannot - # run Linux containers because nested containerisation is disabled. -# - name: Test SSH (Linux only) -# if: runner.os == 'Linux' -# run: | -# sudo service mysql stop # free up port 3306 for ssh tests -# pytest tests/tests_transfers/ssh -# -# - name: Test Google Drive -# env: -# GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} -# GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} -# GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} -# GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} -# run: | -# pytest tests/tests_transfers/gdrive - -# - name: Test AWS -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# AWS_REGION: ${{ secrets.AWS_REGION }} -# AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} -# run: | -# pytest tests/tests_transfers/aws - -# - name: All Other Tests -# run: | -# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws - - name: Install pass on Linux + # this is required for Rclone config encryption if: runner.os == 'Linux' run: | set -euo pipefail @@ -116,16 +86,42 @@ jobs: FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')" pass init "$FPR" - # (Optional) smoke test: ensure pass can encrypt - printf '%s\n' "$(openssl rand -base64 16)" | pass insert -m -f ci/smoke-test + + # run SSH tests only on Linux because Windows and macOS + # are already run within a virtual container and so cannot + # run Linux containers because nested containerisation is disabled. + - name: Test SSH (Linux only) + if: runner.os == 'Linux' + run: | + sudo service mysql stop # free up port 3306 for ssh tests + pytest tests/tests_transfers/ssh + + - name: Test Google Drive + env: + GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} + GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} + GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} + GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} + run: | + pytest tests/tests_transfers/gdrive + +# - name: Test AWS +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# AWS_REGION: ${{ secrets.AWS_REGION }} +# AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} +# run: | +# pytest tests/tests_transfers/aws + +# - name: All Other Tests +# run: | +# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws - name: RClone Encryption run: | - set -euo pipefail - # GNUPGHOME is available here because we wrote it to $GITHUB_ENV pytest -k test_rclone_encryption - build_sdist_wheels: name: Build source distribution needs: [test] diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index c48ffca26..a13db8a72 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -13,7 +13,7 @@ from datashuttle import DataShuttle from datashuttle.configs import load_configs -from datashuttle.utils import aws, gdrive, rclone, ssh, utils +from datashuttle.utils import aws, rclone, ssh, utils class Interface: @@ -568,11 +568,13 @@ def get_rclone_message_for_gdrive_without_browser( ) -> InterfaceOutput: """Get the rclone message for Google Drive setup without a browser.""" try: - output = gdrive.preliminary_for_setup_without_browser( - self.project.cfg, - gdrive_client_secret, - self.project.cfg.rclone.get_rclone_config_name("gdrive"), - log=False, + output = ( + rclone.preliminary_setup_gdrive_config_for_without_browser( + self.project.cfg, + gdrive_client_secret, + self.project.cfg.rclone.get_rclone_config_name("gdrive"), + log=False, + ) ) return True, output except BaseException as e: diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 910066c6e..e52717113 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -21,6 +21,8 @@ Static, ) +from datashuttle.utils import rclone_encryption + class SetupGdriveScreen(ModalScreen): """Dialog window that sets up a Google Drive connection. @@ -294,6 +296,7 @@ async def setup_gdrive_connection_and_update_ui( # This function is called from different screens that # contain different widgets. Therefore, remove all possible # widgets that may / may not be present on the previous screen. + self.show_encryption_screen() for id in [ "#setup_gdrive_cancel_button", "#setup_gdrive_generic_input_box", @@ -336,6 +339,18 @@ def setup_gdrive_connection( # Set encryption on RClone config # ---------------------------------------------------------------------------------- + def show_encryption_screen(self): + """Show the screen asking the user whether to encrypt the Rclone password.""" + message = f"{rclone_encryption.get_explanation_message(self.interface.project.cfg)}" + self.update_message_box_message(message) + + yes_button = Button("Yes", id="setup_gdrive_set_encryption_yes_button") + no_button = Button("No", id="setup_gdrive_set_encryption_no_button") + + self.query_one("#setup_gdrive_buttons_horizontal").mount( + yes_button, no_button + ) + def set_rclone_encryption(self): """Try and encrypt the Rclone config file and inform the user of success / failure.""" success, output = self.interface.try_setup_rclone_encryption() diff --git a/datashuttle/utils/gdrive.py b/datashuttle/utils/gdrive.py index dcb41b401..c2f2fd8bf 100644 --- a/datashuttle/utils/gdrive.py +++ b/datashuttle/utils/gdrive.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -8,71 +7,6 @@ from datashuttle.utils import rclone, utils -# ----------------------------------------------------------------------------- -# Helper Functions -# ----------------------------------------------------------------------------- - -# These functions are used by both API and TUI for setting up connections to google drive. - - -def preliminary_for_setup_without_browser( - cfg: Configs, - gdrive_client_secret: str | None, - rclone_config_name: str, - log: bool = True, -) -> str: - """Prepare rclone configuration for Google Drive without using a browser. - - This function prepares the rclone configuration for Google Drive without using a browser. - - The `config_is_local=false` flag tells rclone that the configuration process is being run - on a headless machine which does not have access to a browser. - - The `--non-interactive` flag is used to control rclone's behaviour while running it through - external applications. An `rclone config create` command would assume default values for config - variables in an interactive mode. If the `--non-interactive` flag is provided and rclone needs - the user to input some detail, a JSON blob will be returned with the question in it. For this - particular setup, rclone outputs a command for user to run on a machine with a browser. - - This function runs `rclone config create` with the user credentials and returns the rclone's output info. - This output info is presented to the user while asking for a `config_token`. - - Next, the user will run rclone's given command, authenticate with google drive and input the - config token given by rclone for datashuttle to proceed with the setup. - """ - client_id_key_value = ( - f"client_id {cfg['gdrive_client_id']} " - if cfg["gdrive_client_id"] - else " " - ) - client_secret_key_value = ( - f"client_secret {gdrive_client_secret} " - if gdrive_client_secret - else "" - ) - output = rclone.call_rclone( - f"config create " - f"{rclone_config_name} " - f"drive " - f"{client_id_key_value}" - f"{client_secret_key_value}" - f"scope drive " - f"root_folder_id {cfg['gdrive_root_folder_id']} " - f"config_is_local=false " - f"--non-interactive", - pipe_std=True, - ) - - # Extracting rclone's message from the json - output_json = json.loads(output.stdout) - message = output_json["Option"]["Help"] - - if log: - utils.log(message) - - return message - - # ----------------------------------------------------------------------------- # Python API # ----------------------------------------------------------------------------- @@ -108,7 +42,7 @@ def prompt_and_get_config_token( with google drive and input the `config_token` generated by rclone. The `config_token` is then used to complete rclone's config setup for google drive. """ - message = preliminary_for_setup_without_browser( + message = rclone.preliminary_setup_gdrive_config_for_without_browser( cfg, gdrive_client_secret, rclone_config_name, log=log ) input_ = utils.get_user_input( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index bce9b14f6..bad5c5bc2 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -9,6 +9,7 @@ from datashuttle.configs.config_class import Configs from datashuttle.utils.custom_types import TopLevelFolder +import json import os import platform import shlex @@ -158,7 +159,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( lambda_func = lambda: process.communicate() stdout, stderr = run_function_that_requires_encrypted_rclone_config_access( - cfg, lambda_func + cfg, lambda_func, check_config_exists=False ) if process.returncode != 0: @@ -171,7 +172,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail( def run_function_that_requires_encrypted_rclone_config_access( - cfg, lambda_func + cfg, lambda_func, check_config_exists: bool = True ) -> Any: """Run command that requires possibly encrypted Rclone config file. @@ -185,7 +186,7 @@ def run_function_that_requires_encrypted_rclone_config_access( cfg.rclone.get_rclone_central_connection_config_filepath() ) - if not rclone_config_filepath.is_file(): + if check_config_exists and not rclone_config_filepath.is_file(): if version.parse(get_datashuttle_version()) <= version.parse("0.7.1"): raise RuntimeError( f"The way RClone configs are managed has changed since version v0.7.1\n" @@ -370,6 +371,71 @@ def setup_rclone_config_for_gdrive( return process +def preliminary_setup_gdrive_config_for_without_browser( + cfg: Configs, + gdrive_client_secret: str | None, + rclone_config_name: str, + log: bool = True, +) -> str: + """Prepare rclone configuration for Google Drive without using a browser. + + This function prepares the rclone configuration for Google Drive without using a browser. + + The `config_is_local=false` flag tells rclone that the configuration process is being run + on a headless machine which does not have access to a browser. + + The `--non-interactive` flag is used to control rclone's behaviour while running it through + external applications. An `rclone config create` command would assume default values for config + variables in an interactive mode. If the `--non-interactive` flag is provided and rclone needs + the user to input some detail, a JSON blob will be returned with the question in it. For this + particular setup, rclone outputs a command for user to run on a machine with a browser. + + This function runs `rclone config create` with the user credentials and returns the rclone's output info. + This output info is presented to the user while asking for a `config_token`. + + Next, the user will run rclone's given command, authenticate with google drive and input the + config token given by rclone for datashuttle to proceed with the setup. + """ + client_id_key_value = ( + f"client_id {cfg['gdrive_client_id']} " + if cfg["gdrive_client_id"] + else " " + ) + client_secret_key_value = ( + f"client_secret {gdrive_client_secret} " + if gdrive_client_secret + else "" + ) + + cfg.rclone.delete_existing_rclone_config_file() + + output = call_rclone( + f"config create " + f"{get_config_arg(cfg)} " + f"{rclone_config_name} " + f"drive " + f"{client_id_key_value}" + f"{client_secret_key_value}" + f"scope drive " + f"root_folder_id {cfg['gdrive_root_folder_id']} " + f"config_is_local=false " + f"--non-interactive", + pipe_std=True, + ) + + try: + # Extracting rclone's message from the json + output_json = json.loads(output.stdout) + message = output_json["Option"]["Help"] + except: + assert False, f"{output.stderr}" + + if log: + utils.log(message) + + return message + + def setup_rclone_config_for_aws( cfg: Configs, rclone_config_name: str, diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index c822551ae..600604a63 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -242,7 +242,7 @@ def run_rclone_config_encrypt(cfg: Configs) -> None: def remove_rclone_encryption(cfg: Configs) -> None: - """Remove encryption from an Rclone config file. + """Remove encryption from a Rclone config file. Set the credentials one last time to remove encryption from the RClone config file. Once removed, clean up the password @@ -322,7 +322,7 @@ def get_windows_password_filepath( def get_explanation_message( cfg: Configs, ) -> str: - """Explaining rclone's default credential storage and OS-specific encryption options. + """Explaining Rclone's default credential storage and OS-specific encryption options. Displayed in both the Python API and the TUI. """ diff --git a/tests/test_utils.py b/tests/test_utils.py index 5cb166cf4..efee425ef 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,6 @@ import asyncio import copy import glob -import json import logging import os import pathlib @@ -14,7 +13,7 @@ from datashuttle import DataShuttle from datashuttle.configs import canonical_configs, canonical_folders -from datashuttle.utils import ds_logger, rclone +from datashuttle.utils import ds_logger # ----------------------------------------------------------------------------- # Setup and Teardown Test Project @@ -433,38 +432,6 @@ def check_config_file(config_path, *kwargs): assert value == config_yaml[name], f"{name}" -# ----------------------------------------------------------------------------- -# Search -# ----------------------------------------------------------------------------- - - -def recursive_search_central(project: DataShuttle): - """ - A convenience function to search project for files on remote folders - using rclone's recursive search. - """ - all_filenames: list[str] = [] - - path_ = (project.cfg["central_path"] / "rawdata").as_posix() - - # -R flag searches recursively - output = rclone.call_rclone( - f"lsjson -R {project.cfg.get_rclone_config_name()}:{path_} {rclone.get_config_arg(project.cfg)}", - pipe_std=True, - ) - - all_files_or_folders = json.loads(output.stdout) - - for file_or_folder in all_files_or_folders: - is_dir = file_or_folder.get("IsDir", False) - - if not is_dir: - file_path = file_or_folder["Path"] - all_filenames.append(f"{path_}/{file_path}") - - return all_filenames - - # ----------------------------------------------------------------------------- # Test Helpers # ----------------------------------------------------------------------------- diff --git a/tests/tests_integration/test_rclone_encryption.py b/tests/tests_integration/test_rclone_encryption.py index cda3f6e24..1e535834d 100644 --- a/tests/tests_integration/test_rclone_encryption.py +++ b/tests/tests_integration/test_rclone_encryption.py @@ -1,3 +1,5 @@ +import os + from datashuttle.utils import rclone_encryption from ..base import BaseTest @@ -6,7 +8,7 @@ class TestRcloneEncryption(BaseTest): def test_set_and_remove_password(self, project): - """""" + """ """ ssh_test_utils.setup_project_for_ssh( project, ) @@ -38,6 +40,8 @@ def test_set_and_remove_password(self, project): rclone_encryption.run_rclone_config_encrypt(project.cfg) + assert "RCLONE_PASSWORD_COMMAND" not in os.environ + with open(rclone_config_path, "r", encoding="utf-8") as f: first_line = f.readline().strip() @@ -45,6 +49,8 @@ def test_set_and_remove_password(self, project): rclone_encryption.remove_rclone_encryption(project.cfg) + assert "RCLONE_PASSWORD_COMMAND" not in os.environ + with open(rclone_config_path, "r", encoding="utf-8") as f: first_line = f.readline().strip() diff --git a/tests/tests_transfers/base_transfer.py b/tests/tests_transfers/base_transfer.py index c01b71141..3957e0d94 100644 --- a/tests/tests_transfers/base_transfer.py +++ b/tests/tests_transfers/base_transfer.py @@ -1,6 +1,7 @@ """ """ import copy +import json import shutil from pathlib import Path @@ -200,7 +201,7 @@ def run_and_check_transfers( # Search the paths that were transferred and tidy them up, # then check against the paths that were expected to be transferred. - transferred_files = test_utils.recursive_search_central(project) + transferred_files = self.recursive_search_central(project) paths_to_transferred_files = self.remove_path_before_rawdata( transferred_files ) @@ -241,8 +242,9 @@ def run_and_check_transfers( # Clean up, removing the temp directories and # resetting the project paths. - rclone.call_rclone( - f"purge {project.cfg.get_rclone_config_name()}:{tmp_central_path.as_posix()} {rclone.get_config_arg(project.cfg)}" + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge {project.cfg.rclone.get_rclone_config_name()}:{tmp_central_path.as_posix()} {rclone.get_config_arg(project.cfg)}", ) shutil.rmtree(tmp_local_path) @@ -256,3 +258,30 @@ def remake_logging_path(self, project): local_path location in the test environment. """ project.get_logging_path().mkdir(parents=True, exist_ok=True) + + @staticmethod + def recursive_search_central(project): + """ + A convenience function to search project for files on remote folders + using rclone's recursive search. + """ + all_filenames: list[str] = [] + + path_ = (project.cfg["central_path"] / "rawdata").as_posix() + + # -R flag searches recursively + output = rclone.call_rclone_for_central_connection( + project.cfg, + f"lsjson -R {project.cfg.rclone.get_rclone_config_name()}:{path_} {rclone.get_config_arg(project.cfg)}", + pipe_std=True, + ) + all_files_or_folders = json.loads(output.stdout) + + for file_or_folder in all_files_or_folders: + is_dir = file_or_folder.get("IsDir", False) + + if not is_dir: + file_path = file_or_folder["Path"] + all_filenames.append(f"{path_}/{file_path}") + + return all_filenames diff --git a/tests/tests_transfers/gdrive/gdrive_test_utils.py b/tests/tests_transfers/gdrive/gdrive_test_utils.py index 99cabe508..747818fcf 100644 --- a/tests/tests_transfers/gdrive/gdrive_test_utils.py +++ b/tests/tests_transfers/gdrive/gdrive_test_utils.py @@ -35,14 +35,24 @@ def setup_gdrive_connection(project: DataShuttle): connection without a browser. The credentials are set in the environment by the CI. To run tests locally, the developer must set them themselves. """ - state = {"first": True} + state = {"count": 0} def mock_input(_: str) -> str: - if state["first"]: - state["first"] = False - return "n" + if state["count"] == 0: + return_value = "n" + state["count"] += 1 + elif state["count"] == 1: + return_value = os.environ["GDRIVE_CONFIG_TOKEN"] + state["count"] += 1 + elif state["count"] == 2: + return_value = "y" + state["count"] += 1 + elif state["count"] == 3: + return_value = "y" else: - return os.environ["GDRIVE_CONFIG_TOKEN"] + raise ValueError(f"return count is {state['count']}") + + return return_value original_input = copy.deepcopy(builtins.input) builtins.input = mock_input # type: ignore diff --git a/tests/tests_transfers/gdrive/test_gdrive_transfer.py b/tests/tests_transfers/gdrive/test_gdrive_transfer.py index fd9effa0d..47fa04fdc 100644 --- a/tests/tests_transfers/gdrive/test_gdrive_transfer.py +++ b/tests/tests_transfers/gdrive/test_gdrive_transfer.py @@ -29,8 +29,9 @@ def gdrive_setup(self, pathtable_and_project): yield [pathtable, project] - rclone.call_rclone( - f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project.project_name}_gdrive:{project.get_central_path()} {rclone.get_config_arg(project.cfg)}", ) @pytest.mark.parametrize( diff --git a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py index e7830005b..fa3bce241 100644 --- a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py +++ b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py @@ -2,6 +2,7 @@ import pytest +from datashuttle import DataShuttle from datashuttle.tui.app import TuiApp from datashuttle.tui.screens.project_manager import ProjectManagerScreen from datashuttle.utils import rclone, utils @@ -31,14 +32,23 @@ def central_path_and_project(self, setup_project_paths): yield central_path, project_name - rclone.call_rclone( - f"purge central_{project_name}_gdrive:{central_path}" # TODO: I think this will fail + project = DataShuttle(project_name) + + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project_name}_gdrive:{central_path} {rclone.get_config_arg(project.cfg)}", ) - @pytest.mark.parametrize("central_path_none", [True, False]) + @pytest.mark.parametrize( + "parameter_sets", + [ + {"central_path_none": True, "set_encryption": True}, + {"central_path_none": False, "set_encryption": False}, + ], + ) @pytest.mark.asyncio async def test_gdrive_connection_setup_without_browser( - self, central_path_none, central_path_and_project + self, parameter_sets, central_path_and_project ): """Test Google Drive connection setup via the TUI. @@ -47,7 +57,14 @@ async def test_gdrive_connection_setup_without_browser( not possible to authenticate via a browser during tests, the connection setup is tested without a browser. The credentials in the environment are set by the CI. For testing locally, the developer must set these themselves. + + We test the case when central path is None or not, and encryption + is set or not. We don't need to test every combination (these settings + are unrelated) so we test across parameter sets. + """ + central_path_none = parameter_sets["central_path_none"] + set_encryption = parameter_sets["set_encryption"] central_path, project_name = central_path_and_project app = TuiApp() @@ -78,8 +95,9 @@ async def test_gdrive_connection_setup_without_browser( "#setup_gdrive_generic_input_box", os.environ["GDRIVE_CONFIG_TOKEN"], ) + await self.scroll_to_click_pause( - pilot, "#setup_gdrive_enter_button" + pilot, "#setup_gdrive_no_browser_enter_button" ) await test_utils.await_task_by_name_if_present( @@ -87,12 +105,36 @@ async def test_gdrive_connection_setup_without_browser( ) assert ( - "Setup Complete!" + "Would you like to encrypt the RClone config file" in pilot.app.screen.query_one( "#gdrive_setup_messagebox_message" ).renderable ) + if set_encryption: + await self.scroll_to_click_pause( + pilot, "#setup_gdrive_set_encryption_yes_button" + ) + + assert ( + "The password was successfully set. Setup complete!" + in pilot.app.screen.query_one( + "#gdrive_setup_messagebox_message" + ).renderable + ) + + else: + await self.scroll_to_click_pause( + pilot, "#setup_gdrive_set_encryption_no_button" + ) + + assert ( + "Setup complete!" + in pilot.app.screen.query_one( + "#gdrive_setup_messagebox_message" + ).renderable + ) + @pytest.mark.asyncio async def test_gdrive_connection_setup_incorrect_config_token( self, setup_project_paths @@ -129,8 +171,9 @@ async def test_gdrive_connection_setup_incorrect_config_token( "#setup_gdrive_generic_input_box", "placeholder", ) + await self.scroll_to_click_pause( - pilot, "#setup_gdrive_enter_button" + pilot, "#setup_gdrive_no_browser_enter_button" ) await test_utils.await_task_by_name_if_present( @@ -180,8 +223,9 @@ async def test_gdrive_connection_setup_incorrect_root_folder_id( "#setup_gdrive_generic_input_box", os.environ["GDRIVE_CONFIG_TOKEN"], ) + await self.scroll_to_click_pause( - pilot, "#setup_gdrive_enter_button" + pilot, "#setup_gdrive_no_browser_enter_button" ) await test_utils.await_task_by_name_if_present( @@ -194,6 +238,7 @@ async def test_gdrive_connection_setup_incorrect_root_folder_id( "#gdrive_setup_messagebox_message" ).renderable ) + assert ( "Error 404: File not found" in pilot.app.screen.query_one( @@ -224,25 +269,30 @@ async def test_cancel_gdrive_connection_setup(self, setup_project_paths): ) # Setup connection and cancel midway + await self.setup_gdrive_connection_via_tui(pilot) + assert ( - "Please authenticate through browser" + "Please authenticate through your browser" in pilot.app.screen.query_one( "#gdrive_setup_messagebox_message" ).renderable ) + await self.scroll_to_click_pause( pilot, "#setup_gdrive_cancel_button" ) # Try setting up the connection again await self.setup_gdrive_connection_via_tui(pilot) + assert ( - "Please authenticate through browser" + "Please authenticate through your browser" in pilot.app.screen.query_one( "#gdrive_setup_messagebox_message" ).renderable ) + await self.scroll_to_click_pause( pilot, "#setup_gdrive_cancel_button" ) @@ -310,7 +360,9 @@ async def setup_gdrive_connection_via_tui( "#setup_gdrive_generic_input_box", os.environ["GDRIVE_CLIENT_SECRET"], ) - await self.scroll_to_click_pause(pilot, "#setup_gdrive_enter_button") + await self.scroll_to_click_pause( + pilot, "#setup_gdrive_no_browser_enter_button" + ) assert ( "Are you running datashuttle on a machine " @@ -321,6 +373,8 @@ async def setup_gdrive_connection_via_tui( ) if with_browser: - await self.scroll_to_click_pause(pilot, "#setup_gdrive_yes_button") + await self.scroll_to_click_pause( + pilot, "#setup_gdrive_has_browser_yes_button" + ) else: await self.scroll_to_click_pause(pilot, "#setup_gdrive_no_button") diff --git a/tests/tests_transfers/ssh/ssh_test_utils.py b/tests/tests_transfers/ssh/ssh_test_utils.py index b57f18fcc..82a14927f 100644 --- a/tests/tests_transfers/ssh/ssh_test_utils.py +++ b/tests/tests_transfers/ssh/ssh_test_utils.py @@ -56,7 +56,7 @@ def setup_ssh_connection(project, setup_ssh_key_pair=True): rclone.setup_rclone_config_for_ssh( project.cfg, - project.cfg.get_rclone_config_name("ssh"), + project.cfg.rclone.get_rclone_config_name("ssh"), private_key_str, ) diff --git a/tests/tests_unit/test_gdrive_preliminary_setup.py b/tests/tests_unit/test_gdrive_preliminary_setup.py index 79924552f..eda806006 100644 --- a/tests/tests_unit/test_gdrive_preliminary_setup.py +++ b/tests/tests_unit/test_gdrive_preliminary_setup.py @@ -4,7 +4,7 @@ import pytest -from datashuttle.utils import gdrive +from datashuttle.utils import rclone class TestGdrivePreliminarySetup: @@ -16,14 +16,14 @@ class TestGdrivePreliminarySetup: def test_preliminary_setup_for_gdrive( self, client_id, root_folder_id, client_secret ): - """Test the outputs of `preliminary_for_setup_without_browser` and check + """Test the outputs of `preliminary_setup_gdrive_config_for_without_browser` and check that they contain the correct credentials in the encoded format. """ mock_configs = { "gdrive_client_id": client_id, "gdrive_root_folder_id": root_folder_id, } - output = gdrive.preliminary_for_setup_without_browser( + output = rclone.preliminary_setup_gdrive_config_for_without_browser( mock_configs, client_secret, "test_gdrive_preliminary" ) From 9a4528426087dacac449b364fafacaee35edc6cb Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 31 Oct 2025 20:32:28 +0000 Subject: [PATCH 048/100] Fixing aws tests. --- tests/tests_transfers/aws/aws_test_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/tests_transfers/aws/aws_test_utils.py b/tests/tests_transfers/aws/aws_test_utils.py index fcc86e772..5cbe37787 100644 --- a/tests/tests_transfers/aws/aws_test_utils.py +++ b/tests/tests_transfers/aws/aws_test_utils.py @@ -34,6 +34,15 @@ def setup_aws_connection(project: DataShuttle): The `AWS_SECRET_ACCESS_KEY` is set in the environment by the CI while testing. For testing locally, the developer must set it themselves. """ + + def mock_input(_: str) -> str: + return "y" + + import builtins + + original_input = copy.deepcopy(builtins.input) + builtins.input = mock_input # type: ignore + original_get_secret = copy.deepcopy(aws.get_aws_secret_access_key) aws.get_aws_secret_access_key = lambda *args, **kwargs: os.environ[ "AWS_SECRET_ACCESS_KEY" @@ -41,6 +50,7 @@ def setup_aws_connection(project: DataShuttle): project.setup_aws_connection() + builtins.input = original_input aws.get_aws_secret_access_key = original_get_secret From 7f1f077ee5df67914f82f8aa81379cf33b013fc8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 31 Oct 2025 21:59:52 +0000 Subject: [PATCH 049/100] Refactor ssh connection set up tests. --- datashuttle/configs/config_class.py | 4 - datashuttle/datashuttle_class.py | 3 +- datashuttle/utils/rclone.py | 11 +-- tests/tests_transfers/ssh/ssh_test_utils.py | 20 +---- tests/tests_transfers/ssh/test_ssh_setup.py | 81 ------------------- .../ssh/test_ssh_suggest_next.py | 1 + 6 files changed, 7 insertions(+), 113 deletions(-) delete mode 100644 tests/tests_transfers/ssh/test_ssh_setup.py diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 638fac9f1..ec47f8dcc 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -169,10 +169,6 @@ def update_config_for_backward_compatability_if_required( if config_dict["connection_method"] is None: config_dict["connection_method"] = "local_only" - def save_rclone_password_state(self): - with open(self.rclone_password_state_file_path, "w") as file: - yaml.dump(self.rclone_has_password, file) - # ------------------------------------------------------------------------- # Utils # ------------------------------------------------------------------------- diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 751c7a5e9..c6a9f3c9c 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1686,7 +1686,8 @@ def _setup_rclone_central_ssh_config( def _setup_rclone_central_local_filesystem_config(self) -> None: rclone.setup_rclone_config_for_local_filesystem( - self.cfg, self.cfg.rclone.get_rclone_config_name("local_filesystem"), + self.cfg, + self.cfg.rclone.get_rclone_config_name("local_filesystem"), ) def _setup_rclone_gdrive_config( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index bad5c5bc2..2498fc771 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -303,6 +303,7 @@ def setup_rclone_config_for_ssh( if log: log_rclone_config_output(cfg) + def setup_rclone_config_for_gdrive( cfg: Configs, rclone_config_name: str, @@ -493,16 +494,6 @@ def setup_rclone_config_for_aws( if log: log_rclone_config_output(cfg) -def delete_existing_rclone_config_file(cfg: Configs): - """ """ - rclone_config_filepath = ( - cfg.rclone.get_rclone_central_connection_config_filepath() - ) - - if rclone_config_filepath.exists(): - rclone_config_filepath.unlink() - cfg.rclone.set_rclone_has_password(False) - def get_config_arg(cfg: Configs) -> str: """Get the full argument to run Rclone commands with a specific config.""" diff --git a/tests/tests_transfers/ssh/ssh_test_utils.py b/tests/tests_transfers/ssh/ssh_test_utils.py index 82a14927f..3ef95796c 100644 --- a/tests/tests_transfers/ssh/ssh_test_utils.py +++ b/tests/tests_transfers/ssh/ssh_test_utils.py @@ -3,7 +3,7 @@ import subprocess import sys -from datashuttle.utils import rclone, ssh, utils +from datashuttle.utils import utils def setup_project_for_ssh( @@ -23,7 +23,7 @@ def setup_project_for_ssh( ) -def setup_ssh_connection(project, setup_ssh_key_pair=True): +def setup_ssh_connection(project): """ Convenience function to verify the server hostkey and ssh key pairs to the Dockerfile image for ssh tests. @@ -46,27 +46,13 @@ def setup_ssh_connection(project, setup_ssh_key_pair=True): orig_isatty = copy.deepcopy(sys.stdin.isatty) sys.stdin.isatty = lambda: True - # Run setup - verified = ssh.verify_ssh_central_host_api( - project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True - ) - - if setup_ssh_key_pair: - private_key_str = ssh.setup_ssh_key_api(project.cfg, log=False) - - rclone.setup_rclone_config_for_ssh( - project.cfg, - project.cfg.rclone.get_rclone_config_name("ssh"), - private_key_str, - ) + project.setup_ssh_connection() # Restore functions builtins.input = orig_builtin utils.get_connection_secret_from_user = orig_get_secret sys.stdin.isatty = orig_isatty - return verified - def docker_is_running(): if not is_docker_installed(): diff --git a/tests/tests_transfers/ssh/test_ssh_setup.py b/tests/tests_transfers/ssh/test_ssh_setup.py deleted file mode 100644 index 10eeb353b..000000000 --- a/tests/tests_transfers/ssh/test_ssh_setup.py +++ /dev/null @@ -1,81 +0,0 @@ -import builtins -import copy -import platform - -import pytest - -from ... import test_utils -from . import ssh_test_utils -from .base_ssh import BaseSSHTransfer - -TEST_SSH = ssh_test_utils.docker_is_running() - - -@pytest.mark.skipif( - platform.system == "Darwin", reason="Docker set up is not robust on macOS." -) -@pytest.mark.skipif( - not TEST_SSH, - reason="SSH tests are not run as docker is either not installed, " - "running or current user is not in the docker group.", -) -class TestSSH(BaseSSHTransfer): - @pytest.fixture(scope="function") - def project(test, tmp_path, setup_ssh_container_fixture): - """Set up a project with configs for SSH into - the test Dockerfile image. - """ - tmp_path = tmp_path / "test with space" - - test_project_name = "test_ssh" - - project = test_utils.setup_project_fixture(tmp_path, test_project_name) - - ssh_test_utils.setup_project_for_ssh( - project, - ) - - yield project - test_utils.teardown_project(project) - - # ----------------------------------------------------------------- - # Test Setup SSH Connection - # ----------------------------------------------------------------- - - @pytest.mark.parametrize("input_", ["n", "o", "@"]) - def test_verify_ssh_central_host_do_not_accept( - self, capsys, project, input_ - ): - """Test that host not accepted if input is not "y".""" - orig_builtin = copy.deepcopy(builtins.input) - builtins.input = lambda _: input_ # type: ignore - - project.setup_ssh_connection() - - builtins.input = orig_builtin - - captured = capsys.readouterr() - - assert "Host not accepted. No connection made.\n" in captured.out - - def test_verify_ssh_central_host_accept(self, capsys, project): - """User is asked to accept the server hostkey. Mock this here - and check hostkey is successfully accepted and written to configs. - """ - test_utils.clear_capsys(capsys) - - verified = ssh_test_utils.setup_ssh_connection( - project, setup_ssh_key_pair=False - ) - - assert verified - captured = capsys.readouterr() - - assert captured.out == "Host accepted.\n" - - with open(project.cfg.hostkeys_path) as file: - hostkey = file.readlines()[0] - - assert ( - f"[{project.cfg['central_host_id']}]:3306 ssh-ed25519 " in hostkey - ) diff --git a/tests/tests_transfers/ssh/test_ssh_suggest_next.py b/tests/tests_transfers/ssh/test_ssh_suggest_next.py index ed6523270..9b87565c4 100644 --- a/tests/tests_transfers/ssh/test_ssh_suggest_next.py +++ b/tests/tests_transfers/ssh/test_ssh_suggest_next.py @@ -27,6 +27,7 @@ def ssh_setup(self, setup_project_paths, setup_ssh_container_fixture): Setup pathtable and project for SSH transfer tests. """ project = test_utils.make_project(setup_project_paths["project_name"]) + ssh_test_utils.setup_project_for_ssh( project, ) From bbadcec98f7c7fb841a83cd52ef4e338c1781859 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sat, 1 Nov 2025 01:25:15 +0000 Subject: [PATCH 050/100] Finalise aws and other tests, revert changes made to setup_aws during erroneous 'fix linting' PR. --- .github/workflows/code_test_and_deploy.yml | 25 +++--- datashuttle/tui/screens/setup_aws.py | 76 ++++++++++++++----- datashuttle/tui/screens/setup_gdrive.py | 2 +- tests/test_utils.py | 7 ++ .../test_rclone_encryption.py | 6 +- .../tests_transfers/aws/test_tui_setup_aws.py | 46 +++++++++-- .../gdrive/test_tui_setup_gdrive.py | 8 +- .../tests_transfers/ssh/test_ssh_transfer.py | 9 +++ 8 files changed, 136 insertions(+), 43 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index e53d47e6d..4e6103eb6 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -86,7 +86,6 @@ jobs: FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')" pass init "$FPR" - # run SSH tests only on Linux because Windows and macOS # are already run within a virtual container and so cannot # run Linux containers because nested containerisation is disabled. @@ -105,22 +104,18 @@ jobs: run: | pytest tests/tests_transfers/gdrive -# - name: Test AWS -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# AWS_REGION: ${{ secrets.AWS_REGION }} -# AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} -# run: | -# pytest tests/tests_transfers/aws - -# - name: All Other Tests -# run: | -# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws + - name: Test AWS + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + run: | + pytest tests/tests_transfers/aws - - name: RClone Encryption + - name: All Other Tests run: | - pytest -k test_rclone_encryption + pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws build_sdist_wheels: name: Build source distribution diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 8527e1dd9..89e247c8e 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -11,6 +11,8 @@ from textual.screen import ModalScreen from textual.widgets import Button, Input, Static +from datashuttle.utils import rclone_encryption + class SetupAwsScreen(ModalScreen): """Dialog window that sets up connection to an Amazon Web Service S3 bucket. @@ -26,7 +28,7 @@ def __init__(self, interface: Interface) -> None: super(SetupAwsScreen, self).__init__() self.interface = interface - self.stage = 0 + self.stage = "init" def compose(self) -> ComposeResult: """Set widgets on the SetupAwsScreen.""" @@ -52,18 +54,35 @@ def on_mount(self) -> None: self.query_one("#setup_aws_secret_access_key_input").visible = False def on_button_pressed(self, event: Button.Pressed) -> None: - """Handle button press on the screen.""" + """Handle button press on the screen. + + The `setup_aws_ok_button` is used for all 'positive' events ('Yes, Ok') + and 'setup_aws_cancel_button' is used for 'negative' events ('No', 'Cancel'). + The appropriate action to take on the button press is determined by the + current stage. + + """ if event.button.id == "setup_aws_cancel_button": - self.dismiss() + if self.stage == "ask_rclone_encryption": + message = "AWS Connection Successful!" # + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_ok_button").label = "Finish" + self.query_one("#setup_aws_cancel_button").remove() + self.stage = "finished" + else: + self.dismiss() - if event.button.id == "setup_aws_ok_button": - if self.stage == 0: + elif event.button.id == "setup_aws_ok_button": + if self.stage == "init": self.prompt_user_for_aws_secret_access_key() - elif self.stage == 1: + elif self.stage == "use_secret_access_key": self.use_secret_access_key_to_setup_aws_connection() - elif self.stage == 2: + elif self.stage == "ask_rclone_encryption": + self.set_rclone_encryption() + + elif self.stage == "finished": self.dismiss() def prompt_user_for_aws_secret_access_key(self) -> None: @@ -73,10 +92,15 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_secret_access_key_input").visible = True - self.stage += 1 + self.query_one("#setup_aws_ok_button") + + self.stage = "use_secret_access_key" def use_secret_access_key_to_setup_aws_connection(self) -> None: - """Set up the AWS connection and inform user of success or failure.""" + """Set up the AWS connection and failure. + + If success, move onto the rclone_encryption screen. + """ secret_access_key = self.query_one( "#setup_aws_secret_access_key_input" ).value @@ -86,11 +110,13 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: ) if success: - message = "AWS Connection Successful!" - self.query_one( - "#setup_aws_secret_access_key_input" - ).visible = False + message = f"{rclone_encryption.get_explanation_message(self.interface.project.cfg)}" + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_secret_access_key_input").remove() + self.query_one("#setup_aws_ok_button").label = "Yes" + self.query_one("#setup_aws_cancel_button").label = "No" + self.stage = "ask_rclone_encryption" else: message = ( f"AWS setup failed. Please check your configs and secret access key" @@ -100,7 +126,23 @@ def use_secret_access_key_to_setup_aws_connection(self) -> None: "#setup_aws_secret_access_key_input" ).disabled = True - self.query_one("#setup_aws_ok_button").label = "Finish" - self.query_one("#setup_aws_messagebox_message").update(message) - self.query_one("#setup_aws_cancel_button").disabled = True - self.stage += 1 + self.query_one("#setup_aws_ok_button").label = "Retry" + self.query_one("#setup_aws_messagebox_message").update(message) + + def set_rclone_encryption(self): + """Try and encrypt the Rclone config file and inform the user of success / failure.""" + success, output = self.interface.try_setup_rclone_encryption() + + if success: + message = ( + "The rclone_encryption was successfully set. Setup complete!" + ) + self.query_one("#setup_aws_messagebox_message").update(message) + self.query_one("#setup_aws_ok_button").label = "Finish" + self.query_one("#setup_aws_cancel_button").remove() + self.stage = "finished" + else: + message = ( + f"The rclone_encryption set up failed. Exception: {output}" + ) + self.query_one("#setup_aws_messagebox_message").update(message) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index e52717113..d62d23c5d 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -357,7 +357,7 @@ def set_rclone_encryption(self): if success: self.set_finish_page( - "The password was successfully set. Setup complete!" + "The encryption was successful. Setup complete!" ) else: message = f"The password set up failed. Exception: {output}" diff --git a/tests/test_utils.py b/tests/test_utils.py index efee425ef..ff9f7849f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -432,6 +432,13 @@ def check_config_file(config_path, *kwargs): assert value == config_yaml[name], f"{name}" +def check_rclone_file_is_encrypted(rclone_config_path): + with open(rclone_config_path, "r", encoding="utf-8") as file: + first_line = file.readline().strip() + + assert first_line == "# Encrypted rclone configuration File" + + # ----------------------------------------------------------------------------- # Test Helpers # ----------------------------------------------------------------------------- diff --git a/tests/tests_integration/test_rclone_encryption.py b/tests/tests_integration/test_rclone_encryption.py index 1e535834d..b066edbfe 100644 --- a/tests/tests_integration/test_rclone_encryption.py +++ b/tests/tests_integration/test_rclone_encryption.py @@ -2,6 +2,7 @@ from datashuttle.utils import rclone_encryption +from .. import test_utils from ..base import BaseTest from ..tests_transfers.ssh import ssh_test_utils @@ -42,10 +43,7 @@ def test_set_and_remove_password(self, project): assert "RCLONE_PASSWORD_COMMAND" not in os.environ - with open(rclone_config_path, "r", encoding="utf-8") as f: - first_line = f.readline().strip() - - assert first_line == "# Encrypted rclone configuration File" + test_utils.check_rclone_file_is_encrypted(rclone_config_path) rclone_encryption.remove_rclone_encryption(project.cfg) diff --git a/tests/tests_transfers/aws/test_tui_setup_aws.py b/tests/tests_transfers/aws/test_tui_setup_aws.py index fc795353b..e184ee4ca 100644 --- a/tests/tests_transfers/aws/test_tui_setup_aws.py +++ b/tests/tests_transfers/aws/test_tui_setup_aws.py @@ -2,10 +2,12 @@ import pytest +from datashuttle import DataShuttle from datashuttle.tui.app import TuiApp from datashuttle.tui.screens.project_manager import ProjectManagerScreen from datashuttle.utils import rclone, utils +from ... import test_utils from ...tests_tui.tui_base import TuiBase from . import aws_test_utils @@ -31,12 +33,18 @@ def central_path_and_project(self, setup_project_paths): yield central_path, project_name - rclone.call_rclone( - f"purge central_{project_name}_aws:{central_path}" - ) # TODO: I think this will fail, needs config + project = DataShuttle(project_name) + + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project_name}_aws:{central_path} {rclone.get_config_arg(project.cfg)}", + ) @pytest.mark.asyncio - async def test_aws_connection_setup(self, central_path_and_project): + @pytest.mark.parametrize("set_encryption", [True, False]) + async def test_aws_connection_setup( + self, central_path_and_project, set_encryption + ): """Test AWS connection setup via the TUI. AWS connection details are filled in the configs tab. The setup @@ -60,12 +68,40 @@ async def test_aws_connection_setup(self, central_path_and_project): ) assert ( - "AWS Connection Successful!" + "Would you like to encrypt the RClone config file" in pilot.app.screen.query_one( "#setup_aws_messagebox_message" ).renderable ) + if set_encryption: + await self.scroll_to_click_pause(pilot, "#setup_aws_ok_button") + + assert ( + "The rclone_encryption was successfully set. Setup complete!" + in pilot.app.screen.query_one( + "#setup_aws_messagebox_message" + ).renderable + ) + + project = pilot.app.screen.interface.project + + test_utils.check_rclone_file_is_encrypted( + project.cfg.rclone.get_rclone_central_connection_config_filepath() + ) + + else: + await self.scroll_to_click_pause( + pilot, "#setup_aws_cancel_button" + ) + + assert ( + "AWS Connection Successful!" + in pilot.app.screen.query_one( + "#setup_aws_messagebox_message" + ).renderable + ) + @pytest.mark.asyncio async def test_aws_connection_setup_failed(self, central_path_and_project): """Test AWS connection setup using an incorrect client secret and check diff --git a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py index fa3bce241..9985ff575 100644 --- a/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py +++ b/tests/tests_transfers/gdrive/test_tui_setup_gdrive.py @@ -117,12 +117,18 @@ async def test_gdrive_connection_setup_without_browser( ) assert ( - "The password was successfully set. Setup complete!" + "The encryption was successful. Setup complete!" in pilot.app.screen.query_one( "#gdrive_setup_messagebox_message" ).renderable ) + project = pilot.app.screen.interface.project + + test_utils.check_rclone_file_is_encrypted( + project.cfg.rclone.get_rclone_central_connection_config_filepath() + ) + else: await self.scroll_to_click_pause( pilot, "#setup_gdrive_set_encryption_no_button" diff --git a/tests/tests_transfers/ssh/test_ssh_transfer.py b/tests/tests_transfers/ssh/test_ssh_transfer.py index a7f74887e..8c995ede9 100644 --- a/tests/tests_transfers/ssh/test_ssh_transfer.py +++ b/tests/tests_transfers/ssh/test_ssh_transfer.py @@ -3,6 +3,7 @@ import pytest +from ... import test_utils from . import ssh_test_utils from .base_ssh import BaseSSHTransfer @@ -157,3 +158,11 @@ def test_ssh_wildcards_3(self, ssh_setup): self.run_and_check_transfers( project, sub_names, ses_names, datatype, expected_transferred_paths ) + + def test_rclone_config_file_encrypted(self, ssh_setup): + """Quick confidence check the set up rclone config is indeed ecrypted.""" + pathtable, project = ssh_setup + + test_utils.check_rclone_file_is_encrypted( + project.cfg.rclone.get_rclone_central_connection_config_filepath() + ) From f4b6d4c119cb76bc903c78ca35531ec55737aa39 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 3 Nov 2025 15:32:11 +0000 Subject: [PATCH 051/100] Add note on docs. --- docs/source/pages/get_started/set-up-a-project.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index 62e121689..0c3193366 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -551,6 +551,8 @@ Running [](setup_aws_connection()) will require entering your (password-protection)= ### Password protecting your connection credentials ++ Add links to this page in the TUI / Api + Datashuttle uses the software `RClone` for all data transfers by default. RClone stores the credentials for connection by default in an unencrypted configuration file. This includes: From c4af1ac17ee8525e100c5063e8f75fa8ed5bab7c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 3 Nov 2025 16:31:31 +0000 Subject: [PATCH 052/100] Fix rebase error introduced. --- datashuttle/tui/screens/setup_gdrive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index d62d23c5d..128568be7 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -292,7 +292,6 @@ async def setup_gdrive_connection_and_update_ui( success, output = worker.result if success: - self.show_password_screen() # This function is called from different screens that # contain different widgets. Therefore, remove all possible # widgets that may / may not be present on the previous screen. From e0c91f1b2fa86ca1373bb6e0bd76a0e7b86e5308 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 3 Nov 2025 23:40:39 +0000 Subject: [PATCH 053/100] Test AWS only because it is hanging in a weird way. --- .github/workflows/code_test_and_deploy.yml | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 4e6103eb6..cf4788412 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -89,20 +89,20 @@ jobs: # run SSH tests only on Linux because Windows and macOS # are already run within a virtual container and so cannot # run Linux containers because nested containerisation is disabled. - - name: Test SSH (Linux only) - if: runner.os == 'Linux' - run: | - sudo service mysql stop # free up port 3306 for ssh tests - pytest tests/tests_transfers/ssh - - - name: Test Google Drive - env: - GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} - GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} - GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} - GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} - run: | - pytest tests/tests_transfers/gdrive +# - name: Test SSH (Linux only) +# if: runner.os == 'Linux' +# run: | +# sudo service mysql stop # free up port 3306 for ssh tests +# pytest tests/tests_transfers/ssh + +# - name: Test Google Drive +# env: +# GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} +# GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} +# GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} +# GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} +# run: | +# pytest tests/tests_transfers/gdrive - name: Test AWS env: @@ -113,9 +113,9 @@ jobs: run: | pytest tests/tests_transfers/aws - - name: All Other Tests - run: | - pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws +# - name: All Other Tests +# run: | +# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws build_sdist_wheels: name: Build source distribution From 1502e1ccd827d799e85f0e4651663a1f436b369d Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 4 Nov 2025 00:12:24 +0000 Subject: [PATCH 054/100] Fix purge. --- tests/tests_transfers/aws/test_aws_suggest_next.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_transfers/aws/test_aws_suggest_next.py b/tests/tests_transfers/aws/test_aws_suggest_next.py index a694a20cf..05c1fef95 100644 --- a/tests/tests_transfers/aws/test_aws_suggest_next.py +++ b/tests/tests_transfers/aws/test_aws_suggest_next.py @@ -27,7 +27,7 @@ def aws_setup(self, setup_project_paths): yield project rclone.call_rclone( - f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent}" + f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" ) @pytest.mark.asyncio From 342b5f581ff023043ec98a7171335183819b70e8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 4 Nov 2025 00:25:48 +0000 Subject: [PATCH 055/100] Try in verbose mode :( --- .github/workflows/code_test_and_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index cf4788412..a8a8dd9c3 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -111,7 +111,7 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} run: | - pytest tests/tests_transfers/aws + pytest tests/tests_transfers/aws -v # - name: All Other Tests # run: | From 4834b7c4adf7d9dbf606d7c428588ba5ffbec95a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 4 Nov 2025 01:08:36 +0000 Subject: [PATCH 056/100] Remove suggest next test and see if this works. --- .../aws/{test_aws_suggest_next.py => _test_aws_suggest_next.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/tests_transfers/aws/{test_aws_suggest_next.py => _test_aws_suggest_next.py} (100%) diff --git a/tests/tests_transfers/aws/test_aws_suggest_next.py b/tests/tests_transfers/aws/_test_aws_suggest_next.py similarity index 100% rename from tests/tests_transfers/aws/test_aws_suggest_next.py rename to tests/tests_transfers/aws/_test_aws_suggest_next.py From 2cf1ef4b39a5cee1754c205fdcb02c44419511d8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 4 Nov 2025 01:21:42 +0000 Subject: [PATCH 057/100] Finally I think have the test fixes. --- .../aws/{_test_aws_suggest_next.py => test_aws_suggest_next.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/tests_transfers/aws/{_test_aws_suggest_next.py => test_aws_suggest_next.py} (88%) diff --git a/tests/tests_transfers/aws/_test_aws_suggest_next.py b/tests/tests_transfers/aws/test_aws_suggest_next.py similarity index 88% rename from tests/tests_transfers/aws/_test_aws_suggest_next.py rename to tests/tests_transfers/aws/test_aws_suggest_next.py index 05c1fef95..a67028bb0 100644 --- a/tests/tests_transfers/aws/_test_aws_suggest_next.py +++ b/tests/tests_transfers/aws/test_aws_suggest_next.py @@ -27,7 +27,7 @@ def aws_setup(self, setup_project_paths): yield project rclone.call_rclone( - f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" + f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" ) @pytest.mark.asyncio From 2b42a4628e64e09a9eb976e1edde73953f132e34 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 16:51:03 +0000 Subject: [PATCH 058/100] Remove tests again for checking. --- .../aws/{test_aws_suggest_next.py => _test_aws_suggest_next.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/tests_transfers/aws/{test_aws_suggest_next.py => _test_aws_suggest_next.py} (100%) diff --git a/tests/tests_transfers/aws/test_aws_suggest_next.py b/tests/tests_transfers/aws/_test_aws_suggest_next.py similarity index 100% rename from tests/tests_transfers/aws/test_aws_suggest_next.py rename to tests/tests_transfers/aws/_test_aws_suggest_next.py From 210c57c85d44e1e9a78000a531e3a9882fbb4501 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 17:05:49 +0000 Subject: [PATCH 059/100] Try removing another test. --- .../aws/{test_tui_setup_aws.py => _test_tui_setup_aws.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/tests_transfers/aws/{test_tui_setup_aws.py => _test_tui_setup_aws.py} (100%) diff --git a/tests/tests_transfers/aws/test_tui_setup_aws.py b/tests/tests_transfers/aws/_test_tui_setup_aws.py similarity index 100% rename from tests/tests_transfers/aws/test_tui_setup_aws.py rename to tests/tests_transfers/aws/_test_tui_setup_aws.py From 74310efa82ec33948b63c8fffab72e3ad2a0b383 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 17:15:35 +0000 Subject: [PATCH 060/100] Try again. --- .../{_test_aws_suggest_next.py => test_aws_suggest_next.py} | 5 +++-- tests/tests_transfers/aws/test_aws_transfer.py | 5 +++-- .../aws/{_test_tui_setup_aws.py => test_tui_setup_aws.py} | 0 tests/tests_transfers/gdrive/test_gdrive_suggest_next.py | 5 +++-- 4 files changed, 9 insertions(+), 6 deletions(-) rename tests/tests_transfers/aws/{_test_aws_suggest_next.py => test_aws_suggest_next.py} (93%) rename tests/tests_transfers/aws/{_test_tui_setup_aws.py => test_tui_setup_aws.py} (100%) diff --git a/tests/tests_transfers/aws/_test_aws_suggest_next.py b/tests/tests_transfers/aws/test_aws_suggest_next.py similarity index 93% rename from tests/tests_transfers/aws/_test_aws_suggest_next.py rename to tests/tests_transfers/aws/test_aws_suggest_next.py index a67028bb0..5edca41f6 100644 --- a/tests/tests_transfers/aws/_test_aws_suggest_next.py +++ b/tests/tests_transfers/aws/test_aws_suggest_next.py @@ -26,8 +26,9 @@ def aws_setup(self, setup_project_paths): yield project - rclone.call_rclone( - f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}", ) @pytest.mark.asyncio diff --git a/tests/tests_transfers/aws/test_aws_transfer.py b/tests/tests_transfers/aws/test_aws_transfer.py index a03af4657..cd73004bf 100644 --- a/tests/tests_transfers/aws/test_aws_transfer.py +++ b/tests/tests_transfers/aws/test_aws_transfer.py @@ -27,8 +27,9 @@ def aws_setup(self, pathtable_and_project): yield [pathtable, project] - rclone.call_rclone( - f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}" + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}", ) @pytest.mark.parametrize( diff --git a/tests/tests_transfers/aws/_test_tui_setup_aws.py b/tests/tests_transfers/aws/test_tui_setup_aws.py similarity index 100% rename from tests/tests_transfers/aws/_test_tui_setup_aws.py rename to tests/tests_transfers/aws/test_tui_setup_aws.py diff --git a/tests/tests_transfers/gdrive/test_gdrive_suggest_next.py b/tests/tests_transfers/gdrive/test_gdrive_suggest_next.py index 39a98a6ae..6655ca7d8 100644 --- a/tests/tests_transfers/gdrive/test_gdrive_suggest_next.py +++ b/tests/tests_transfers/gdrive/test_gdrive_suggest_next.py @@ -28,8 +28,9 @@ def gdrive_setup(self, setup_project_paths): yield project - rclone.call_rclone( - f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent}" + rclone.call_rclone_for_central_connection( + project.cfg, + f"purge central_{project.project_name}_gdrive:{project.cfg['central_path'].parent} {rclone.get_config_arg(project.cfg)}", ) @pytest.mark.asyncio From e90077a803e60348c265fb20d801a36a8be4e225 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 17:26:33 +0000 Subject: [PATCH 061/100] Revert CI changes. --- .github/workflows/code_test_and_deploy.yml | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index a8a8dd9c3..8c43e7c93 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -89,20 +89,20 @@ jobs: # run SSH tests only on Linux because Windows and macOS # are already run within a virtual container and so cannot # run Linux containers because nested containerisation is disabled. -# - name: Test SSH (Linux only) -# if: runner.os == 'Linux' -# run: | -# sudo service mysql stop # free up port 3306 for ssh tests -# pytest tests/tests_transfers/ssh - -# - name: Test Google Drive -# env: -# GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} -# GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} -# GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} -# GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} -# run: | -# pytest tests/tests_transfers/gdrive + - name: Test SSH (Linux only) + if: runner.os == 'Linux' + run: | + sudo service mysql stop # free up port 3306 for ssh tests + pytest tests/tests_transfers/ssh + + - name: Test Google Drive + env: + GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} + GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} + GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} + GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} + run: | + pytest tests/tests_transfers/gdrive - name: Test AWS env: @@ -113,9 +113,9 @@ jobs: run: | pytest tests/tests_transfers/aws -v -# - name: All Other Tests -# run: | -# pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws + - name: All Other Tests + run: | + pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws build_sdist_wheels: name: Build source distribution From 2e0f793901a273e3ef82e2e8035b55dbcf9f1ead Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 18:13:24 +0000 Subject: [PATCH 062/100] Update documentation. --- .github/workflows/code_test_and_deploy.yml | 2 +- .../pages/get_started/set-up-a-project.md | 89 +++++++++++++++---- 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 8c43e7c93..4e6103eb6 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -111,7 +111,7 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} run: | - pytest tests/tests_transfers/aws -v + pytest tests/tests_transfers/aws - name: All Other Tests run: | diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index 0c3193366..1bcd5b923 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -547,35 +547,86 @@ project.setup_aws_connection() Running [](setup_aws_connection()) will require entering your `AWS Secret Access Key` and the setup will be completed. +:::: + (password-protection)= -### Password protecting your connection credentials +# Password protecting your connection credentials -+ Add links to this page in the TUI / Api +Datashuttle uses [RClone](https://rclone.org/) for all data transfers by default. +RClone stores connection credentials (such as SSH keys or API tokens) in a local configuration file that, by default, is not encrypted. -Datashuttle uses the software `RClone` for all data transfers by default. -RClone stores the credentials for connection by default in an unencrypted configuration file. -This includes: +This file can include: -- ssh connection: the private SSH key -- Google Drive: some API thing -- Amazon S3: Access key ID and XXX +- SSH connections: your private SSH key +- Google Drive connections: your OAuth access token and client secret +- Amazon S3 connections: your AWS Access Key ID and Secret Access Key By default, these are stored in your home directory which should be secure. However, for an -additional layer of security, it is possible to encrpy the Rclone config file. +additional layer of security, it is possible to encrypt the Rclone config file using the +system credential manager of your operating system. This file will then be +non-readable for anyone who does not have access to your machine user account. Note that +anyone with access to the machine user account will be able to decrypt the Rclone file. + +Despite this layer of security, it is not recommended to use datashuttle for remote connectivity on +a machine to which you do not have secure access, even with user account encryption of the RClone config. -When setting up the connection, datashuttle will offer the option to set the RClone configuration. -This automatically uses the system credential manager: +For details on setting up encryption, see the section below. On Windows, you will +need to be running in PowerShell, and on Linux you will need `pass` package installed. -Windows : (requires powershell) -macOS : set up -Linux : requires pass +::::{tab-set} -This means the file is only uncryptable on your local machine or user) CHECK USER. +:::{tab-item} Windows -TODO: think more about the credentials file... its' stupid to have this itself plain text in datashuttle? +On Windows, Datashuttle uses the PowerShell `PSCredential` system to encrypt the RClone config file. -Despite this layer of security, it is not recommended to use datashuttle for remote connectivity on -a machine to which you do not have secure access, even with password protection of the RClone config. +- A random password is generated and stored as a `.clixml` credential file under a `credentials` folder in the project config location. +- The password can only be decrypted by the same Windows user account that created it. +- The encryption and decryption process uses PowerShell, so PowerShell must be available (it will not work from `cmd.exe`). + +When encryption is enabled, RClone automatically retrieves the password from the PSCredential store whenever it runs. + +::: + +:::{tab-item} macOS + +On macOS, Datashuttle uses the built-in Keychain via the `security` command-line tool. + +- A random password is generated using `openssl rand -base64 40`. +- The password is securely stored in your login Keychain under the service name corresponding to your RClone config. +- Only your macOS user account can access this key. + +When you first set up encryption, macOS may prompt you to authorize access to the Keychain. +Once approved, RClone will automatically retrieve the key when needed. + +::: + +:::{tab-item} Linux + +1. Install `pass`: + ```bash + sudo apt install pass + ``` +2. Initialize the password store with your GPG key: + ```bash + pass init + ``` + +Once initialized, Datashuttle will: +- Generate a random password with `openssl rand -base64 40` +- Store it securely in the GPG-encrypted password store +- Configure RClone to retrieve it automatically with: + ```bash + /usr/bin/pass + ``` + +::: + +:::: + +## Removing encryption + +Encryption of the rclone config used for the central connection (either SSH, Google Drive or AWS) +can be removed with the following command: -TODO: test if `pass` is not installed on linux that the error is propagated to the TUI properly +[](remove_rclone_encryption()) From f40bc3353a360996ead51deb9e166653f085de7b Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 5 Nov 2025 18:27:05 +0000 Subject: [PATCH 063/100] Quick fix for running on local filesystem mode. --- datashuttle/utils/rclone.py | 17 +++++++++++++---- tests/tests_integration/test_create_folders.py | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 2498fc771..a466af915 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -115,9 +115,12 @@ def call_rclone_through_script_for_central_connection( shell=False, ) - output = run_function_that_requires_encrypted_rclone_config_access( - cfg, lambda_func - ) + if cfg["connection_method"] in ["ssh", "gdrive", "aws"]: + output = run_function_that_requires_encrypted_rclone_config_access( + cfg, lambda_func + ) + else: + output = lambda_func() if output.returncode != 0: prompt_rclone_download_if_does_not_exist() @@ -662,9 +665,15 @@ def transfer_data( f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error ) - if cfg.rclone.get_rclone_config_encryption_state(): + if ( + cfg["connection_method"] in ["ssh", "aws", "gdrive"] + and cfg.rclone.get_rclone_config_encryption_state() + ): # TODO: this is a quick and dirty fix but this MUST be handled better rclone_encryption.remove_credentials_as_password_command() + # 1) now 'for central connection' terminology is confused, one is for all and the other checks internally if it is aws or not. This is okay but must be consistent + # 2) make a utils function to do the connection method check, this is still kind of weird / error prone + return output diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index 72102d986..a14c6b45d 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -309,7 +309,9 @@ def test_all_top_level_folders(self, project, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) - def test_get_next_sub(self, project, return_with_prefix, top_level_folder): + def test_get_next_sub__( + self, project, return_with_prefix, top_level_folder + ): """Test that the next subject number is suggested correctly. This takes the union of subjects available in the local and central repository. As such test the case where either are From 8f3e444e6ee6973529ec0056713dba3aac8b2021 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 6 Nov 2025 14:40:28 +0000 Subject: [PATCH 064/100] Extend Mock Configs class. --- .../test_gdrive_preliminary_setup.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/tests_unit/test_gdrive_preliminary_setup.py b/tests/tests_unit/test_gdrive_preliminary_setup.py index eda806006..50b794ae2 100644 --- a/tests/tests_unit/test_gdrive_preliminary_setup.py +++ b/tests/tests_unit/test_gdrive_preliminary_setup.py @@ -19,10 +19,27 @@ def test_preliminary_setup_for_gdrive( """Test the outputs of `preliminary_setup_gdrive_config_for_without_browser` and check that they contain the correct credentials in the encoded format. """ - mock_configs = { - "gdrive_client_id": client_id, - "gdrive_root_folder_id": root_folder_id, - } + from collections import UserDict + from pathlib import Path + + class MockConfigs(UserDict): + def __init__(self, client_id_, root_folder_id_): + super(MockConfigs, self).__init__() + self.data["gdrive_client_id"] = client_id_ + self.data["gdrive_root_folder_id"] = root_folder_id_ + self.data["connection_method"] = "drive" + + class RClone: + def delete_existing_rclone_config_file(self): + pass + + def get_rclone_central_connection_config_filepath(self): + return Path("") + + self.rclone = RClone() + + mock_configs = MockConfigs(client_id, root_folder_id) + output = rclone.preliminary_setup_gdrive_config_for_without_browser( mock_configs, client_secret, "test_gdrive_preliminary" ) From b31694776c24b9667719255d10bd1cece430cdac Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 18 Nov 2025 21:25:47 +0000 Subject: [PATCH 065/100] Fixing tests. --- datashuttle/tui/tabs/create_folders.py | 10 +++++--- datashuttle/utils/rclone.py | 4 ++- datashuttle/utils/rclone_encryption.py | 8 +++--- tests/base.py | 2 +- .../tests_integration/test_local_only_mode.py | 2 +- tests/tests_integration/test_logging.py | 4 +-- .../tests_integration/test_search_methods.py | 25 +++++++++++-------- .../test_backwards_compatibility.py | 2 +- tests/tests_transfers/base_transfer.py | 2 +- tests/tests_tui/tui_base.py | 2 +- 10 files changed, 37 insertions(+), 24 deletions(-) diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index a82613d86..5363ac8f1 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -42,6 +42,7 @@ require_double_click, ) from datashuttle.tui.utils.tui_validators import NeuroBlueprintValidator +from datashuttle.utils import rclone_encryption class CreateFoldersTab(TreeAndInputTab): @@ -338,9 +339,12 @@ def suggest_next_sub_ses( in canonical_configs.get_connection_methods_list() ) - if include_central and self.interface.project.cfg[ - "connection_method" - ] in ["aws", "gdrive", "ssh"]: + if ( + include_central + and rclone_encryption.connection_method_requires_encryption( + self.interface.project.cfg["connection_method"] + ) + ): self.searching_central_popup_widget = ( SearchingCentralForNextSubSesPopup(prefix) ) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a466af915..6ff077673 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -504,7 +504,9 @@ def get_config_arg(cfg: Configs) -> str: cfg.rclone.get_rclone_central_connection_config_filepath() ) - if cfg["connection_method"] in ["aws", "gdrive", "ssh"]: + if rclone_encryption.connection_method_requires_encryption( + cfg["connection_method"] + ): return f'--config "{rclone_config_path}"' else: return "" diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 600604a63..0cebd8084 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -304,13 +304,15 @@ def remove_credentials_as_password_command(): os.environ.pop("RCLONE_PASSWORD_COMMAND") +def connection_method_requires_encryption(connection_method: str): + return connection_method in ["aws", "gdrive", "ssh"] + + def get_windows_password_filepath( cfg: Configs, ) -> Path: """Get the canonical location where datashuttle stores the windows credentials.""" - assert cfg["connection_method"] in ["aws", "gdrive", "ssh"], ( - "password should only be set for ssh, aws, gdrive." - ) + assert connection_method_requires_encryption(cfg["connection_method"]) base_path = canonical_folders.get_datashuttle_path() / "credentials" diff --git a/tests/base.py b/tests/base.py index f0e45588b..1fd9106b7 100644 --- a/tests/base.py +++ b/tests/base.py @@ -4,7 +4,7 @@ from . import test_utils -TEST_PROJECT_NAME = "test_project" +TEST_PROJECT_NAME = "ds-unique-test-project-d375gd234vds2f" class BaseTest: diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 9bef8af16..eb461984b 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -9,7 +9,7 @@ from .. import test_utils from ..base import BaseTest -TEST_PROJECT_NAME = "test_project" +TEST_PROJECT_NAME = "ds-unique-test-project-d375gd234vds2f" class TestLocalOnlyProject(BaseTest): diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 9b5bd4a0b..5f58591f5 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -86,7 +86,7 @@ def clean_project_name(self): Switch on datashuttle logging as required for these tests, then turn back off during tear-down. """ - project_name = "test_project" + project_name = "ds-unique-test-project-d375gd234vds2f" test_utils.delete_project_if_it_exists(project_name) test_utils.set_datashuttle_loggers(disable=False) @@ -290,7 +290,7 @@ def test_logs_upload_and_download( assert "Using config file from" in log assert "--include" in log assert "sub-11/ses-123/anat/**" in log - assert "/central/test_project/rawdata" in log + assert "/central/ds-unique-test-project-d375gd234vds2f/rawdata" in log @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_logs_upload_and_download_folder_or_file( diff --git a/tests/tests_integration/test_search_methods.py b/tests/tests_integration/test_search_methods.py index a53953146..7f28733ac 100644 --- a/tests/tests_integration/test_search_methods.py +++ b/tests/tests_integration/test_search_methods.py @@ -19,7 +19,7 @@ class TestSubSesSearches(BaseTest): @pytest.mark.parametrize("return_full_path", [True, False]) def test_local_vs_central_search_methods( - self, project, monkeypatch, return_full_path + self, project, monkeypatch, return_full_path, tmp_path ): """ Test the `search_local_filesystem` and `search_central_via_connection` @@ -55,18 +55,23 @@ def test_local_vs_central_search_methods( test_utils.write_file(central_path / path_.parent.parent.parent / f"{path_.parent.parent.name}.md", contents="hello_world",) # fmt: on - # Monkeycatch `get_rclone_config_name` to return `local` and set `local` - # as a rclone config entry associated with the local filesystem. By this - # method we can hijack `search_central_via_connection` to run locally - # (though it is set up in practice to run via ssh, gdrive or aws). + # search_central_via_connection will run the transfer + # function but with additional checks for rclone password + # through `run_function_that_requires_encrypted_rclone_config_access`. + # Here we monkeypatch that to skip all of those checks. + call_rclone(r"config create local local nounc true") + + from datashuttle.utils import rclone + + def mock_rclone_caller(_, func, optional=None): + return func() + monkeypatch.setattr( - project.cfg, - "get_rclone_config_name", - lambda connection_method: "local", + rclone, + "run_function_that_requires_encrypted_rclone_config_access", + mock_rclone_caller, ) - call_rclone("config create local local nounc true") - # Perform a range of checks across folders and files # and check the outputs of both approaches match. # fmt: off diff --git a/tests/tests_regression/test_backwards_compatibility.py b/tests/tests_regression/test_backwards_compatibility.py index cb23165d0..9c1f40f26 100644 --- a/tests/tests_regression/test_backwards_compatibility.py +++ b/tests/tests_regression/test_backwards_compatibility.py @@ -6,7 +6,7 @@ from .. import test_utils -TEST_PROJECT_NAME = "test_project" +TEST_PROJECT_NAME = "ds-unique-test-project-d375gd234vds2f" class TestBackwardsCompatibility: diff --git a/tests/tests_transfers/base_transfer.py b/tests/tests_transfers/base_transfer.py index 3957e0d94..3dc15e51b 100644 --- a/tests/tests_transfers/base_transfer.py +++ b/tests/tests_transfers/base_transfer.py @@ -32,7 +32,7 @@ def pathtable_and_project(self, tmpdir_factory): tmp_path = tmpdir_factory.mktemp("test") base_path = tmp_path / "test with space" - test_project_name = "test_file_conflicts" + test_project_name = "ds-unique-test-project-d375gd234vds2f" project = test_utils.setup_project_fixture( base_path, test_project_name diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index a0c8f16d6..8bc443ce9 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -43,7 +43,7 @@ async def empty_project_paths(self, tmp_path_factory, monkeypatch): 2) It fails for testing CLI, because CLI spawns a new process in which `get_datashuttle_path()` is not monkeypatched. """ - project_name = "my-test-project" + project_name = "ds-unique-test-project-d375gd234vds2f" tmp_path = tmp_path_factory.mktemp("test") tmp_config_path = tmp_path / "config" From 60b30beaf086d3ab32b2e3eee9f0ce0914b0258c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 18 Nov 2025 21:43:15 +0000 Subject: [PATCH 066/100] Fix validate_from_path. --- datashuttle/configs/canonical_folders.py | 5 +++++ datashuttle/datashuttle_functions.py | 8 ++++---- tests/tests_integration/test_validation.py | 11 +++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index d9426bcb7..13716e4e2 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -82,6 +82,11 @@ def get_datashuttle_path() -> Path: return Path.home() / ".datashuttle" +def get_internal_datashuttle_from_path(): + """Get a placeholder path for `validate_project_from_path()`.""" + return get_datashuttle_path() / "_datashuttle_from_path" + + def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: """Return the datashuttle config path for the project. diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 2d618bb30..57d4bf67b 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -13,9 +13,7 @@ Optional, ) -from datashuttle.configs import ( - canonical_configs, -) +from datashuttle.configs import canonical_configs, canonical_folders from datashuttle.configs.config_class import Configs from datashuttle.utils import ( validation, @@ -85,6 +83,8 @@ def validate_project_from_path( # Create some mock configs for the validation call, # then for each top-level folder, run the validation + # Note `get_internal_datashuttle_from_path` generates a placeholder + # folder path but this is not actually created. placeholder_configs = { key: None for key in canonical_configs.get_canonical_configs().keys() } @@ -92,7 +92,7 @@ def validate_project_from_path( cfg = Configs( project_name=project_path.name, - file_path=None, # type: ignore + file_path=canonical_folders.get_internal_datashuttle_from_path(), # type: ignore input_dict=placeholder_configs, ) diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 34c5a02fd..f9be7d12b 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -5,6 +5,7 @@ import pytest from datashuttle import validate_project_from_path +from datashuttle.configs import canonical_folders from datashuttle.utils import formatting, validation from datashuttle.utils.custom_exceptions import NeuroBlueprintError @@ -787,7 +788,7 @@ def test_name_templates_validate_project(self, project): # Test Quick Validation Function # ---------------------------------------------------------------------------------- - def test_quick_validation(self, mocker, project): + def test_validate_project_from_path(self, mocker, project): project.create_folders("rawdata", "sub-1") os.makedirs(project.cfg["local_path"] / "rawdata" / "sub-02") project.create_folders("derivatives", "sub-1") @@ -803,6 +804,12 @@ def test_quick_validation(self, mocker, project): assert "VALUE_LENGTH" in str(w[1].message) assert len(w) == 2 + # This is used internally to generate a Configs class, + # but should not actually be written to. + assert ( + not canonical_folders.get_internal_datashuttle_from_path().exists() + ) + # For good measure, monkeypatch and change all defaults, # ensuring they are propagated to the validate_project # function (which is tested above) @@ -824,7 +831,7 @@ def test_quick_validation(self, mocker, project): assert kwargs["top_level_folder_list"] == ["derivatives"] assert kwargs["name_templates"] == {"on": False} - def test_quick_validation_top_level_folder(self, project): + def test_validate_from_path_top_level_folder(self, project): """Test that errors are raised as expected on bad project path input. """ From fd91242f6de4f6d829c6e0fc0e66613b14a7e0a9 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 18 Nov 2025 21:54:19 +0000 Subject: [PATCH 067/100] Fix perform_rclone_check. --- datashuttle/utils/rclone.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 6ff077673..6af166332 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -774,15 +774,26 @@ def perform_rclone_check( "central", top_level_folder ).parent.as_posix() - output = call_rclone_for_central_connection( - cfg, - f"{rclone_args('check')} " - f'"{local_filepath}" ' - f'"{cfg.rclone.get_rclone_config_name()}:{central_filepath}"' - f"{get_config_arg(cfg)} " - f"--combined -", - pipe_std=True, - ) + if rclone_encryption.connection_method_requires_encryption( + cfg["connection_method"] + ): + output = call_rclone_for_central_connection( + cfg, + f"{rclone_args('check')} " + f'"{local_filepath}" ' + f'"{cfg.rclone.get_rclone_config_name()}:{central_filepath}" ' + f"--combined - " + f"{get_config_arg(cfg)}", + pipe_std=True, + ) + else: + output = call_rclone( + f"{rclone_args('check')} " + f'"{local_filepath}" ' + f'"{cfg.rclone.get_rclone_config_name()}:{central_filepath}" ' + f"--combined - ", + pipe_std=True, + ) return output.stdout.decode("utf-8") From 3b0d6ee73bc9e4ab758c4803588661be7a1c6799 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Tue, 18 Nov 2025 22:04:39 +0000 Subject: [PATCH 068/100] Fix linting. --- datashuttle/utils/rclone_encryption.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 0cebd8084..168572c67 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -305,6 +305,7 @@ def remove_credentials_as_password_command(): def connection_method_requires_encryption(connection_method: str): + """Check whether the connection method stores sensitive information.""" return connection_method in ["aws", "gdrive", "ssh"] From d51d94ea848a4bb5588b4710a2000a3858201106 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 19 Nov 2025 12:02:51 +0000 Subject: [PATCH 069/100] Some tidy up and extend use cases for new encrpytion checker. --- datashuttle/configs/rclone_configs.py | 26 ++++++++++---------------- datashuttle/utils/rclone.py | 18 ++++++++---------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 151681d57..7e9748549 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -12,7 +12,7 @@ import yaml from datashuttle.configs import canonical_folders -from datashuttle.utils import utils +from datashuttle.utils import rclone_encryption, utils class RCloneConfigs: @@ -51,11 +51,9 @@ def load_rclone_config_is_encrypted(self) -> dict: called a lot, we track this explicitly when a rclone config is encrypted / unencrypted and store to disk between sessions. """ - assert self.datashuttle_configs["connection_method"] in [ - "ssh", - "aws", - "gdrive", - ] + assert rclone_encryption.connection_method_requires_encryption( + self.datashuttle_configs["connection_method"] + ) if self.rclone_encryption_state_file_path.is_file(): with open(self.rclone_encryption_state_file_path, "r") as file: @@ -78,11 +76,9 @@ def set_rclone_config_encryption_state(self, value: bool) -> None: Note that this is stored to disk each call (rather than tracked locally) to ensure it is updated live if updated through the Python API while the TUI is also running. """ - assert self.datashuttle_configs["connection_method"] in [ - "ssh", - "aws", - "gdrive", - ] + assert rclone_encryption.connection_method_requires_encryption( + self.datashuttle_configs["connection_method"] + ) rclone_config_is_encrypted = self.load_rclone_config_is_encrypted() @@ -97,11 +93,9 @@ def get_rclone_config_encryption_state( self, ) -> dict: """Return whether the config file associated with the current `connection_method`.""" - assert self.datashuttle_configs["connection_method"] in [ - "ssh", - "aws", - "gdrive", - ] + assert rclone_encryption.connection_method_requires_encryption( + self.datashuttle_configs["connection_method"] + ) rclone_config_is_encrypted = self.load_rclone_config_is_encrypted() diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 6af166332..d8d05b8f1 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -115,7 +115,9 @@ def call_rclone_through_script_for_central_connection( shell=False, ) - if cfg["connection_method"] in ["ssh", "gdrive", "aws"]: + if rclone_encryption.connection_method_requires_encryption( + cfg["connection_method"] + ): output = run_function_that_requires_encrypted_rclone_config_access( cfg, lambda_func ) @@ -553,7 +555,9 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None: def get_rclone_config_filepath(cfg: Configs) -> Path: """Get the path to the central Rclone config for the current `connection_method`.""" - if cfg["connection_method"] in ["aws", "ssh", "gdrive"]: + if rclone_encryption.connection_method_requires_encryption( + cfg["connection_method"] + ): config_filepath = ( cfg.rclone.get_rclone_central_connection_config_filepath() ) @@ -656,7 +660,7 @@ def transfer_data( cfg, f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.rclone.get_rclone_config_name()}:' - f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error + f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', ) elif upload_or_download == "download": @@ -664,15 +668,9 @@ def transfer_data( cfg, f"{rclone_args('copy')} " f'"{cfg.rclone.get_rclone_config_name()}:' - f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', # TODO: handle the error + f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', ) - if ( - cfg["connection_method"] in ["ssh", "aws", "gdrive"] - and cfg.rclone.get_rclone_config_encryption_state() - ): # TODO: this is a quick and dirty fix but this MUST be handled better - rclone_encryption.remove_credentials_as_password_command() - # 1) now 'for central connection' terminology is confused, one is for all and the other checks internally if it is aws or not. This is okay but must be consistent # 2) make a utils function to do the connection method check, this is still kind of weird / error prone From 4ed8a8acbcfeccecd85e1867f97c514b3c8d0a8d Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 19 Nov 2025 23:02:23 +0000 Subject: [PATCH 070/100] Apply some fixes from self review 1. --- datashuttle/configs/canonical_folders.py | 13 ++++--- datashuttle/configs/config_class.py | 4 +- datashuttle/configs/rclone_configs.py | 48 ++++++++---------------- datashuttle/datashuttle_class.py | 2 +- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/rclone.py | 29 +++++++++++++- 6 files changed, 54 insertions(+), 44 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index 13716e4e2..e4d94d1d2 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -82,7 +82,7 @@ def get_datashuttle_path() -> Path: return Path.home() / ".datashuttle" -def get_internal_datashuttle_from_path(): +def get_internal_datashuttle_from_path() -> Path: """Get a placeholder path for `validate_project_from_path()`.""" return get_datashuttle_path() / "_datashuttle_from_path" @@ -100,16 +100,19 @@ def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: return base_path, temp_logs_path -def get_rclone_config_base_path(): +def get_rclone_config_base_path() -> Path: """Return the path to the Rclone config file. This is used for RClone config files for transfer targets (ssh, aws, gdrive). This should match where RClone itself stores the config by default, as described here: https://rclone.org/docs/#config-string - Because RClone's resolution is a little complex, in some rare cases the - below may not match where RClone stores its configs. This just means that - local filesystem configs and transfer configs are stored in a separate place, + Because RClone's resolution process for where it stores its config files + is a little complex, in some rare cases the below may not match where + RClone stores its configs. This just means that local filesystem configs, + which are stored in the default `rclone.conf` file for backwards compatibility + reasons. and transfer configs which are stored in their own file at the + path returned from this function, are stored in a separate places, which is not a huge deal. """ if platform.system() == "Windows": diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index ec47f8dcc..6ece95673 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -37,8 +37,8 @@ def __init__( ) -> None: """Initialize the Configs class with project name, file path, and config dictionary. - This class also holds `RCloneConfigs` under the `.rclone` attribute to manage - the configs + This class also holds `RCloneConfigs` that manage the Rclone config files + used for transfer. Parameters ---------- diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 7e9748549..91ae7db0f 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -1,42 +1,44 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from pathlib import Path - from datashuttle.utils.custom_types import ( - OverwriteExistingFiles, - ) + from datashuttle.configs.configs_class import Configs import yaml from datashuttle.configs import canonical_folders -from datashuttle.utils import rclone_encryption, utils +from datashuttle.utils import rclone_encryption class RCloneConfigs: """Class to manage the RClone configuration file. This is a file that RClone creates to hold all information about local and - remote transfer targets. For example, the ssh RClone config holds the private key. + central transfer targets. For example, the SSH RClone config holds the private key, + the GDrive rclone config holds the access token, etc. In datashuttle, local filesystem configs uses the Rclone default configuration file, - that RClone manages. However, remote transfers to ssh, aws and gdrive are held in - separate config files (set using RClone's --config argument). Then being separate - means these files can be separately encrypted. + that RClone manages, for backwards comatability reasons. However, SSH, AWS and GDrive + configs are stored in separate config files (set using RClone's --config argument). + Then being separate means these files can be separately encrypted. This class tracks the state on whether a RClone config is encrypted, as well as provides the default names for the rclone conf (e.g. central__). Parameters ---------- + datashuttle_configs + Parent Configs class. + config_base_class Path to the datashuttle configs folder where all configs for the project are stored. """ - def __init__(self, datashuttle_configs, config_base_path): + def __init__(self, datashuttle_configs: Configs, config_base_path: Path): """Construct the class.""" self.datashuttle_configs = datashuttle_configs self.rclone_encryption_state_file_path = ( @@ -73,8 +75,9 @@ def load_rclone_config_is_encrypted(self) -> dict: def set_rclone_config_encryption_state(self, value: bool) -> None: """Store the current state of the rclone config encryption for the `connection_method`. - Note that this is stored to disk each call (rather than tracked locally) to ensure - it is updated live if updated through the Python API while the TUI is also running. + Note that this is stored to disk each call (rather than tracked in memory) + to ensure it is updated properly if changed through the Python API + while the TUI is also running. """ assert rclone_encryption.connection_method_requires_encryption( self.datashuttle_configs["connection_method"] @@ -119,27 +122,6 @@ def get_rclone_central_connection_config_filepath(self) -> Path: / f"{self.get_rclone_config_name()}.conf" ) - def make_rclone_transfer_options( - self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool - ) -> Dict: - """Create a dictionary of rclone transfer options.""" - allowed_overwrite = ["never", "always", "if_source_newer"] - - if overwrite_existing_files not in allowed_overwrite: - utils.log_and_raise_error( - f"`overwrite_existing_files` not " - f"recognised, must be one of: " - f"{allowed_overwrite}", - ValueError, - ) - - return { - "overwrite_existing_files": overwrite_existing_files, - "show_transfer_progress": True, - "transfer_verbosity": "vv", - "dry_run": dry_run, - } - def delete_existing_rclone_config_file(self) -> None: """Delete the Rclone config file if it exists.""" rclone_config_filepath = ( diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index c6a9f3c9c..adc077a76 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -790,7 +790,7 @@ def _transfer_specific_file_or_folder( upload_or_download, top_level_folder, include_list, - self.cfg.rclone.make_rclone_transfer_options( + rclone.make_rclone_transfer_options( overwrite_existing_files, dry_run ), ) diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 983c1e1c0..cd294670e 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -99,7 +99,7 @@ def __init__( self.__upload_or_download, self.__top_level_folder, include_list, - cfg.rclone.make_rclone_transfer_options( + rclone.make_rclone_transfer_options( overwrite_existing_files, dry_run ), ) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d8d05b8f1..756e78e65 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -7,7 +7,10 @@ from subprocess import CompletedProcess from datashuttle.configs.config_class import Configs - from datashuttle.utils.custom_types import TopLevelFolder + from datashuttle.utils.custom_types import ( + OverwriteExistingFiles, + TopLevelFolder, + ) import json import os @@ -635,7 +638,7 @@ def transfer_data( rclone_options A list of options to pass to Rclone's copy function. - see `cfg.rclone.make_rclone_transfer_options()`. + see `make_rclone_transfer_options()`. Returns ------- @@ -677,6 +680,28 @@ def transfer_data( return output +def make_rclone_transfer_options( + overwrite_existing_files: OverwriteExistingFiles, dry_run: bool +) -> Dict: + """Create a dictionary of rclone transfer options.""" + allowed_overwrite = ["never", "always", "if_source_newer"] + + if overwrite_existing_files not in allowed_overwrite: + utils.log_and_raise_error( + f"`overwrite_existing_files` not " + f"recognised, must be one of: " + f"{allowed_overwrite}", + ValueError, + ) + + return { + "overwrite_existing_files": overwrite_existing_files, + "show_transfer_progress": True, + "transfer_verbosity": "vv", + "dry_run": dry_run, + } + + def get_local_and_central_file_differences( cfg: Configs, top_level_folders_to_check: List[TopLevelFolder], From 2c644a2f3ba38f2b1886d993090223b5d55cfd7a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Wed, 19 Nov 2025 23:05:42 +0000 Subject: [PATCH 071/100] Rename to rclone_file_is_encrypted. --- datashuttle/configs/rclone_configs.py | 2 +- datashuttle/datashuttle_class.py | 10 +++++----- datashuttle/utils/rclone.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index 91ae7db0f..aab15d9c1 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -92,7 +92,7 @@ def set_rclone_config_encryption_state(self, value: bool) -> None: with open(self.rclone_encryption_state_file_path, "w") as file: yaml.dump(rclone_config_is_encrypted, file) - def get_rclone_config_encryption_state( + def rclone_file_is_encrypted( self, ) -> dict: """Return whether the config file associated with the current `connection_method`.""" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index adc077a76..4b98b8fb4 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -841,7 +841,7 @@ def setup_ssh_connection(self) -> None: f"{self.cfg.rclone.get_rclone_central_connection_config_filepath()}.\n" ) - if not self.cfg.rclone.get_rclone_config_encryption_state(): + if not self.cfg.rclone.rclone_file_is_encrypted(): if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() @@ -910,7 +910,7 @@ def setup_gdrive_connection(self) -> None: self.cfg, process, log=True ) - if not self.cfg.rclone.get_rclone_config_encryption_state(): + if not self.cfg.rclone.rclone_file_is_encrypted(): if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() @@ -950,7 +950,7 @@ def setup_aws_connection(self) -> None: self._setup_rclone_aws_config(aws_secret_access_key, log=True) - if not self.cfg.rclone.get_rclone_config_encryption_state(): + if not self.cfg.rclone.rclone_file_is_encrypted(): if self._ask_user_rclone_encryption(): self._try_encrypt_rclone_config() @@ -1004,7 +1004,7 @@ def _try_encrypt_rclone_config( def encrypt_rclone_config(self) -> None: """Encrypt the rclone config file for the central connection.""" - if self.cfg.rclone.get_rclone_config_encryption_state(): + if self.cfg.rclone.rclone_file_is_encrypted(): raise RuntimeError( "This config file is already encrypted. " "First, use `remove_rclone_encryption` to remove it." @@ -1016,7 +1016,7 @@ def encrypt_rclone_config(self) -> None: def remove_rclone_encryption(self) -> None: """Unencrypt the rclone config file for the central connection.""" - if not self.cfg.rclone.get_rclone_config_encryption_state(): + if not self.cfg.rclone.rclone_file_is_encrypted(): raise RuntimeError( f"The config for the current connection method: " f"{self.cfg['connection_method']} " diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 756e78e65..c14c046cd 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -207,7 +207,7 @@ def run_function_that_requires_encrypted_rclone_config_access( f"Please set up the {cfg['connection_method']} connection again." ) - is_encrypted = cfg.rclone.get_rclone_config_encryption_state() + is_encrypted = cfg.rclone.rclone_file_is_encrypted() if is_encrypted: rclone_encryption.set_credentials_as_password_command(cfg) From 31c80b04b3172b84eb27ed19599b6d0e89007412 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 00:26:19 +0000 Subject: [PATCH 072/100] Remove unecessary type hint ignore. --- datashuttle/datashuttle_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 57d4bf67b..e4b57e015 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -92,7 +92,7 @@ def validate_project_from_path( cfg = Configs( project_name=project_path.name, - file_path=canonical_folders.get_internal_datashuttle_from_path(), # type: ignore + file_path=canonical_folders.get_internal_datashuttle_from_path(), input_dict=placeholder_configs, ) From ccb718bd6d326964bdc44c447c1d5db289871563 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 00:29:29 +0000 Subject: [PATCH 073/100] Rename preliminary_setup_gdrive_config_for_without_browser. --- datashuttle/tui/interface.py | 12 +++++------- datashuttle/utils/gdrive.py | 2 +- datashuttle/utils/rclone.py | 2 +- tests/tests_unit/test_gdrive_preliminary_setup.py | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index a13db8a72..bff4d1b42 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -568,13 +568,11 @@ def get_rclone_message_for_gdrive_without_browser( ) -> InterfaceOutput: """Get the rclone message for Google Drive setup without a browser.""" try: - output = ( - rclone.preliminary_setup_gdrive_config_for_without_browser( - self.project.cfg, - gdrive_client_secret, - self.project.cfg.rclone.get_rclone_config_name("gdrive"), - log=False, - ) + output = rclone.preliminary_setup_gdrive_config_without_browser( + self.project.cfg, + gdrive_client_secret, + self.project.cfg.rclone.get_rclone_config_name("gdrive"), + log=False, ) return True, output except BaseException as e: diff --git a/datashuttle/utils/gdrive.py b/datashuttle/utils/gdrive.py index c2f2fd8bf..499817c46 100644 --- a/datashuttle/utils/gdrive.py +++ b/datashuttle/utils/gdrive.py @@ -42,7 +42,7 @@ def prompt_and_get_config_token( with google drive and input the `config_token` generated by rclone. The `config_token` is then used to complete rclone's config setup for google drive. """ - message = rclone.preliminary_setup_gdrive_config_for_without_browser( + message = rclone.preliminary_setup_gdrive_config_without_browser( cfg, gdrive_client_secret, rclone_config_name, log=log ) input_ = utils.get_user_input( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index c14c046cd..0cdbe0bb2 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -380,7 +380,7 @@ def setup_rclone_config_for_gdrive( return process -def preliminary_setup_gdrive_config_for_without_browser( +def preliminary_setup_gdrive_config_without_browser( cfg: Configs, gdrive_client_secret: str | None, rclone_config_name: str, diff --git a/tests/tests_unit/test_gdrive_preliminary_setup.py b/tests/tests_unit/test_gdrive_preliminary_setup.py index 50b794ae2..a77faece0 100644 --- a/tests/tests_unit/test_gdrive_preliminary_setup.py +++ b/tests/tests_unit/test_gdrive_preliminary_setup.py @@ -16,7 +16,7 @@ class TestGdrivePreliminarySetup: def test_preliminary_setup_for_gdrive( self, client_id, root_folder_id, client_secret ): - """Test the outputs of `preliminary_setup_gdrive_config_for_without_browser` and check + """Test the outputs of `preliminary_setup_gdrive_config_without_browser` and check that they contain the correct credentials in the encoded format. """ from collections import UserDict @@ -40,7 +40,7 @@ def get_rclone_central_connection_config_filepath(self): mock_configs = MockConfigs(client_id, root_folder_id) - output = rclone.preliminary_setup_gdrive_config_for_without_browser( + output = rclone.preliminary_setup_gdrive_config_without_browser( mock_configs, client_secret, "test_gdrive_preliminary" ) From ee695644d9c5c54821dd37de03f28556fc8b5f1c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 01:00:06 +0000 Subject: [PATCH 074/100] Small tidy ups. --- datashuttle/tui/screens/setup_aws.py | 2 +- datashuttle/tui/screens/setup_gdrive.py | 2 +- datashuttle/utils/rclone.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 89e247c8e..8df42cfaf 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -64,7 +64,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: """ if event.button.id == "setup_aws_cancel_button": if self.stage == "ask_rclone_encryption": - message = "AWS Connection Successful!" # + message = "AWS Connection Successful!" self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" self.query_one("#setup_aws_cancel_button").remove() diff --git a/datashuttle/tui/screens/setup_gdrive.py b/datashuttle/tui/screens/setup_gdrive.py index 128568be7..a70e816b2 100644 --- a/datashuttle/tui/screens/setup_gdrive.py +++ b/datashuttle/tui/screens/setup_gdrive.py @@ -29,7 +29,7 @@ class SetupGdriveScreen(ModalScreen): If the config contains a "gdrive_client_id", the user is prompted to enter a client secret. If the user has access to a browser, a Google Drive - authentication page will open. Otherwise, the user is asked to run a rclone command + authentication page will open. Otherwise, the user is asked to run an Rclone command and input a config token. """ diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 0cdbe0bb2..6b6b8af7b 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -393,7 +393,7 @@ def preliminary_setup_gdrive_config_without_browser( The `config_is_local=false` flag tells rclone that the configuration process is being run on a headless machine which does not have access to a browser. - The `--non-interactive` flag is used to control rclone's behaviour while running it through + The `--non-interactive` flag is used to control Rclone's behaviour while running it through external applications. An `rclone config create` command would assume default values for config variables in an interactive mode. If the `--non-interactive` flag is provided and rclone needs the user to input some detail, a JSON blob will be returned with the question in it. For this From 1700651bf81cdf3e2fd6ba745706e960ff83970f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 01:14:16 +0000 Subject: [PATCH 075/100] Revert change to create_folders conditional. --- datashuttle/tui/tabs/create_folders.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 5363ac8f1..a82613d86 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -42,7 +42,6 @@ require_double_click, ) from datashuttle.tui.utils.tui_validators import NeuroBlueprintValidator -from datashuttle.utils import rclone_encryption class CreateFoldersTab(TreeAndInputTab): @@ -339,12 +338,9 @@ def suggest_next_sub_ses( in canonical_configs.get_connection_methods_list() ) - if ( - include_central - and rclone_encryption.connection_method_requires_encryption( - self.interface.project.cfg["connection_method"] - ) - ): + if include_central and self.interface.project.cfg[ + "connection_method" + ] in ["aws", "gdrive", "ssh"]: self.searching_central_popup_widget = ( SearchingCentralForNextSubSesPopup(prefix) ) From 7cea970187ece182710224de63ddeafb6c1e04d4 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 01:28:54 +0000 Subject: [PATCH 076/100] Small refactor. --- datashuttle/utils/rclone.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 6b6b8af7b..73372ac5c 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -505,13 +505,13 @@ def setup_rclone_config_for_aws( def get_config_arg(cfg: Configs) -> str: """Get the full argument to run Rclone commands with a specific config.""" - rclone_config_path = ( - cfg.rclone.get_rclone_central_connection_config_filepath() - ) - if rclone_encryption.connection_method_requires_encryption( cfg["connection_method"] ): + rclone_config_path = ( + cfg.rclone.get_rclone_central_connection_config_filepath() + ) + return f'--config "{rclone_config_path}"' else: return "" From 2e3a71085ca41732426636607796de38bed4be11 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 01:37:46 +0000 Subject: [PATCH 077/100] Remove --ask-password. --- datashuttle/utils/rclone.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 73372ac5c..a77d91092 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -663,7 +663,7 @@ def transfer_data( cfg, f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.rclone.get_rclone_config_name()}:' - f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', + f'{central_filepath}" {extra_arguments} {get_config_arg(cfg)}', ) elif upload_or_download == "download": @@ -671,12 +671,9 @@ def transfer_data( cfg, f"{rclone_args('copy')} " f'"{cfg.rclone.get_rclone_config_name()}:' - f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)} --ask-password=false', + f'{central_filepath}" "{local_filepath}" {extra_arguments} {get_config_arg(cfg)}', ) - # 1) now 'for central connection' terminology is confused, one is for all and the other checks internally if it is aws or not. This is okay but must be consistent - # 2) make a utils function to do the connection method check, this is still kind of weird / error prone - return output From ccbad2a55a3ad278b13621945af1ec061eec3f1c Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:19:26 +0000 Subject: [PATCH 078/100] Update datashuttle/utils/rclone_encryption.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/utils/rclone_encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 168572c67..d146d2869 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -235,7 +235,7 @@ def run_rclone_config_encrypt(cfg: Configs) -> None: raise RuntimeError( f"\n--- STDOUT ---\n{output.stdout}\n" f"\n--- STDERR ---\n{output.stderr}\n" - "Could not remove the password from the RClone config. See the error message above." + "Could not encrypt the RClone config. See the error message above." ) remove_credentials_as_password_command() From 94614fc032433cabb56cff3a83865db2acf39350 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:20:00 +0000 Subject: [PATCH 079/100] Update datashuttle/utils/rclone.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/utils/rclone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a77d91092..bb7a6eecd 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -145,7 +145,7 @@ def call_rclone_with_popen( Killing a process might be required when running rclone setup in a thread worker to allow the user to cancel the setup process. In such a case, cancelling the thread worker alone will not kill the rclone process, so we need to kill the - env process explicitly. + process explicitly. """ command = "rclone " + command process = subprocess.Popen( From cec3c509ca9aa867d0c9a52d2c7bedede48b2bd9 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 15:23:53 +0000 Subject: [PATCH 080/100] Improve a test docstring. --- tests/tests_integration/test_rclone_encryption.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/tests_integration/test_rclone_encryption.py b/tests/tests_integration/test_rclone_encryption.py index b066edbfe..50d9dd03c 100644 --- a/tests/tests_integration/test_rclone_encryption.py +++ b/tests/tests_integration/test_rclone_encryption.py @@ -1,4 +1,5 @@ import os +import textwrap from datashuttle.utils import rclone_encryption @@ -8,8 +9,13 @@ class TestRcloneEncryption(BaseTest): - def test_set_and_remove_password(self, project): - """ """ + def test_set_and_remove_rclone_config_encryption(self, project): + """Test that RClone config encryption is set up and torn down correctly. + + First, we generate a mock RClone config file or a mock SSH project. + Then we check datashuttle functions for encryption, decryption and + cleaning up environment variables work as expected. + """ ssh_test_utils.setup_project_for_ssh( project, ) @@ -21,8 +27,6 @@ def test_set_and_remove_password(self, project): if rclone_config_path.exists(): rclone_config_path.unlink() - import textwrap - config_content = textwrap.dedent(f"""\ [{project.cfg.rclone.get_rclone_config_name()}] type = sftp @@ -49,6 +53,7 @@ def test_set_and_remove_password(self, project): assert "RCLONE_PASSWORD_COMMAND" not in os.environ + # Read the file contents to check it is no longer encrypted. with open(rclone_config_path, "r", encoding="utf-8") as f: first_line = f.readline().strip() From 1991857dae2258516099a2c763c6af0010b0117e Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 15:25:42 +0000 Subject: [PATCH 081/100] Remove debugging clause. --- datashuttle/utils/rclone.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a77d91092..bf8785cac 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -432,12 +432,9 @@ def preliminary_setup_gdrive_config_without_browser( pipe_std=True, ) - try: - # Extracting rclone's message from the json - output_json = json.loads(output.stdout) - message = output_json["Option"]["Help"] - except: - assert False, f"{output.stderr}" + # Extracting rclone's message from the json + output_json = json.loads(output.stdout) + message = output_json["Option"]["Help"] if log: utils.log(message) From 0f50c1b4f00983db050607c4bc28125eb6a614d9 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:25:58 +0000 Subject: [PATCH 082/100] Update datashuttle/configs/rclone_configs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/configs/rclone_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index aab15d9c1..bd83b7b69 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -21,7 +21,7 @@ class RCloneConfigs: the GDrive rclone config holds the access token, etc. In datashuttle, local filesystem configs uses the Rclone default configuration file, - that RClone manages, for backwards comatability reasons. However, SSH, AWS and GDrive + that RClone manages, for backwards compatibility reasons. However, SSH, AWS and GDrive configs are stored in separate config files (set using RClone's --config argument). Then being separate means these files can be separately encrypted. From 611efb259449d45542a4f6c139949aebcd5be882 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:35:19 +0000 Subject: [PATCH 083/100] =?UTF-8?q?Fix=20typo=20in=20test=20docstring:=20'?= =?UTF-8?q?ecrypted'=20=E2=86=92=20'encrypted'=20(#637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * Fix typo: 'ecrypted' → 'encrypted' in test docstring Co-authored-by: JoeZiminski <55797454+JoeZiminski@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: JoeZiminski <55797454+JoeZiminski@users.noreply.github.com> --- tests/tests_transfers/ssh/test_ssh_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_transfers/ssh/test_ssh_transfer.py b/tests/tests_transfers/ssh/test_ssh_transfer.py index 8c995ede9..72b310a81 100644 --- a/tests/tests_transfers/ssh/test_ssh_transfer.py +++ b/tests/tests_transfers/ssh/test_ssh_transfer.py @@ -160,7 +160,7 @@ def test_ssh_wildcards_3(self, ssh_setup): ) def test_rclone_config_file_encrypted(self, ssh_setup): - """Quick confidence check the set up rclone config is indeed ecrypted.""" + """Quick confidence check the set up rclone config is indeed encrypted.""" pathtable, project = ssh_setup test_utils.check_rclone_file_is_encrypted( From ed1a89de17aa40d31b89cfdfbe1d953e8979d29c Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:36:24 +0000 Subject: [PATCH 084/100] Update datashuttle/configs/rclone_configs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/configs/rclone_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index bd83b7b69..dfe41ba9f 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -94,7 +94,7 @@ def set_rclone_config_encryption_state(self, value: bool) -> None: def rclone_file_is_encrypted( self, - ) -> dict: + ) -> bool: """Return whether the config file associated with the current `connection_method`.""" assert rclone_encryption.connection_method_requires_encryption( self.datashuttle_configs["connection_method"] From f5f44da67feda678afe2bfedffeb0e60e32a647b Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:36:43 +0000 Subject: [PATCH 085/100] Update datashuttle/utils/rclone_encryption.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/utils/rclone_encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index d146d2869..8dbb90a8b 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -337,7 +337,7 @@ def get_explanation_message( pass_type = { "ssh": "your private SSH key", - "aws": "your IAM access key ID and seceret access key", + "aws": "your IAM access key ID and secret access key", "gdrive": "your Google Drive access token and client secret (if set)", } From b1a5032161ffca9d04be80fd559684f066c0911c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 15:37:31 +0000 Subject: [PATCH 086/100] Fix broken try / except. --- datashuttle/utils/rclone_encryption.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 168572c67..dd44aef7d 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -92,23 +92,22 @@ def set_password_linux(cfg: Configs) -> None: "`pass` is required to set password. Install e.g. sudo apt install pass." ) - try: - output = subprocess.run( - ["pass", "ls"], - shell=True, - capture_output=True, - text=True, - ) - except subprocess.CalledProcessError as e: - if "pass init" in e.stderr: + output = subprocess.run( + "pass ls", + shell=True, + capture_output=True, + text=True, + ) + if output.returncode != 0: + if "pass init" in output.stderr: raise RuntimeError( "Password store is not initialized. " "Run `pass init ` before using `pass`." ) else: raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}", - f"\n--- STDERR ---\n{output.stderr}", + f"\n--- STDOUT ---\n{output.stdout}" + f"\n--- STDERR ---\n{output.stderr}" "Could not set up password with `pass`. See the error message above.", ) From 06b4a93916ee6e7cc3e1ba347c7b85f12f268629 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 15:45:00 +0000 Subject: [PATCH 087/100] Replace direct raise with utils function. --- datashuttle/utils/rclone_encryption.py | 66 +++++++++++++++----------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index c5a6f3ba3..de2d1f67f 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -46,9 +46,10 @@ def set_password_windows(cfg: Configs) -> None: password_filepath.unlink() shell = shutil.which("powershell") - if not shell: - raise RuntimeError( - "powershell.exe not found in PATH (need Windows PowerShell 5.1)." + if shell is None: + utils.log_and_raise_error( + "powershell.exe not found in PATH (need Windows PowerShell 5.1).", + RuntimeError, ) ps_cmd = ( @@ -59,15 +60,16 @@ def set_password_windows(cfg: Configs) -> None: ) output = subprocess.run( - [shell, "-NoProfile", "-Command", ps_cmd], + [shell, "-NoProfile", "-Command", ps_cmd], # type: ignore capture_output=True, text=True, ) if output.returncode != 0: - raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}", - f"\n--- STDERR ---\n{output.stderr}", + utils.log_and_raise_error( + f"\n--- STDOUT ---\n{output.stdout}" + f"\n--- STDERR ---\n{output.stderr}" "Could not set the PSCredential with System.web. See the error message above.", + RuntimeError, ) @@ -88,8 +90,9 @@ def set_password_linux(cfg: Configs) -> None: text=True, ) if output.returncode != 0: - raise RuntimeError( - "`pass` is required to set password. Install e.g. sudo apt install pass." + utils.log_and_raise_error( + "`pass` is required to set password. Install e.g. sudo apt install pass.", + RuntimeError, ) output = subprocess.run( @@ -100,15 +103,17 @@ def set_password_linux(cfg: Configs) -> None: ) if output.returncode != 0: if "pass init" in output.stderr: - raise RuntimeError( + utils.log_and_raise_error( "Password store is not initialized. " - "Run `pass init ` before using `pass`." + "Run `pass init ` before using `pass`.", + RuntimeError, ) else: - raise RuntimeError( + utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" "Could not set up password with `pass`. See the error message above.", + RuntimeError, ) output = subprocess.run( @@ -118,10 +123,11 @@ def set_password_linux(cfg: Configs) -> None: text=True, ) if output.returncode != 0: - raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}", - f"\n--- STDERR ---\n{output.stderr}", + utils.log_and_raise_error( + f"\n--- STDOUT ---\n{output.stdout}" + f"\n--- STDERR ---\n{output.stderr}" "Could not remove the password from the RClone config. See the error message above.", + RuntimeError, ) @@ -142,10 +148,11 @@ def set_password_macos(cfg: Configs) -> None: ) if output.returncode != 0: - raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}", - f"\n--- STDERR ---\n{output.stderr}", - "Could not remove the password from the RClone config. See the error message above.", + utils.log_and_raise_error( + f"\n--- STDOUT ---\n{output.stdout}" + f"\n--- STDERR ---\n{output.stderr}" + "Could not encrypt the RClone config. See the error message above.", + RuntimeError, ) @@ -172,7 +179,9 @@ def set_credentials_as_password_command(cfg: Configs) -> None: shell = shutil.which("powershell") if not shell: - raise RuntimeError("powershell.exe not found in PATH") + utils.log_and_raise_error( + "powershell.exe not found in PATH", RuntimeError + ) # Escape single quotes inside PowerShell string by doubling them cmd = ( @@ -215,9 +224,10 @@ def run_rclone_config_encrypt(cfg: Configs) -> None: if not rclone_config_path.exists(): connection_method = cfg["connection_method"] - raise RuntimeError( + utils.log_and_raise_error( f"Rclone config file for: {connection_method} was not found. " - f"Make sure you set up the connection first with `setup_{connection_method}_connection()`" + f"Make sure you set up the connection first with `setup_{connection_method}_connection()`", + RuntimeError, ) save_credentials_password(cfg) @@ -231,10 +241,11 @@ def run_rclone_config_encrypt(cfg: Configs) -> None: text=True, ) if output.returncode != 0: - raise RuntimeError( + utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}\n" f"\n--- STDERR ---\n{output.stderr}\n" - "Could not encrypt the RClone config. See the error message above." + "Could not encrypt the RClone config. See the error message above.", + RuntimeError, ) remove_credentials_as_password_command() @@ -260,10 +271,11 @@ def remove_rclone_encryption(cfg: Configs) -> None: text=True, ) if output.returncode != 0: - raise RuntimeError( - f"\n--- STDOUT ---\n{output.stdout}", - f"\n--- STDERR ---\n{output.stderr}", + utils.log_and_raise_error( + f"\n--- STDOUT ---\n{output.stdout}" + f"\n--- STDERR ---\n{output.stderr}" "Could not remove the password from the RClone config. See the error message above.", + RuntimeError, ) remove_credentials_as_password_command() From 6fdb3572c8a53ef34047236910511c768e9f6863 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 15:50:44 +0000 Subject: [PATCH 088/100] Fixes to rclone_encryption.py --- datashuttle/utils/rclone_encryption.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index de2d1f67f..67d1ec50e 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -17,7 +17,6 @@ import shutil import subprocess -from datashuttle.configs import canonical_folders from datashuttle.utils import utils @@ -126,7 +125,7 @@ def set_password_linux(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could not remove the password from the RClone config. See the error message above.", + "Could encrypt the password from the RClone config. See the error message above.", RuntimeError, ) @@ -326,7 +325,8 @@ def get_windows_password_filepath( """Get the canonical location where datashuttle stores the windows credentials.""" assert connection_method_requires_encryption(cfg["connection_method"]) - base_path = canonical_folders.get_datashuttle_path() / "credentials" + # Put this folder next to the project (datashuttle) config file + base_path = cfg.file_path.parent / "credentials" base_path.mkdir(exist_ok=True, parents=True) From e873b760d01553c34d9c8cc39044401c453455b7 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 16:14:55 +0000 Subject: [PATCH 089/100] Small fixes rclone encryption. --- datashuttle/utils/rclone_encryption.py | 43 ++++++++++++++------------ 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 67d1ec50e..939a70e92 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -67,7 +67,7 @@ def set_password_windows(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could not set the PSCredential with System.web. See the error message above.", + "\nCould not set the PSCredential with System.web. See the error message above.", RuntimeError, ) @@ -111,7 +111,7 @@ def set_password_linux(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could not set up password with `pass`. See the error message above.", + "\nCould not set up password with `pass`. See the error message above.", RuntimeError, ) @@ -125,7 +125,7 @@ def set_password_linux(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could encrypt the password from the RClone config. See the error message above.", + "\nCould not encrypt the RClone config. See the error message above.", RuntimeError, ) @@ -150,7 +150,7 @@ def set_password_macos(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could not encrypt the RClone config. See the error message above.", + "\nCould not encrypt the RClone config. See the error message above.", RuntimeError, ) @@ -233,21 +233,22 @@ def run_rclone_config_encrypt(cfg: Configs) -> None: set_credentials_as_password_command(cfg) - output = subprocess.run( - f"rclone config encryption set --config {rclone_config_path.as_posix()}", - shell=True, - capture_output=True, - text=True, - ) - if output.returncode != 0: - utils.log_and_raise_error( - f"\n--- STDOUT ---\n{output.stdout}\n" - f"\n--- STDERR ---\n{output.stderr}\n" - "Could not encrypt the RClone config. See the error message above.", - RuntimeError, + try: + output = subprocess.run( + f"rclone config encryption set --config {rclone_config_path.as_posix()}", + shell=True, + capture_output=True, + text=True, ) - - remove_credentials_as_password_command() + if output.returncode != 0: + utils.log_and_raise_error( + f"\n--- STDOUT ---\n{output.stdout}\n" + f"\n--- STDERR ---\n{output.stderr}\n" + "\nCould not encrypt the RClone config. See the error message above.", + RuntimeError, + ) + finally: + remove_credentials_as_password_command() def remove_rclone_encryption(cfg: Configs) -> None: @@ -273,14 +274,16 @@ def remove_rclone_encryption(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could not remove the password from the RClone config. See the error message above.", + "\nCould not remove the password from the RClone config. See the error message above.", RuntimeError, ) remove_credentials_as_password_command() if platform.system() == "Windows": - get_windows_password_filepath(cfg).unlink() + password_filepath = get_windows_password_filepath(cfg) + if password_filepath.exists(): + password_filepath.unlink() elif platform.system() == "Linux": name = cfg.rclone.get_rclone_config_name() From 3d2cd66699476a3e6eade9d0fd84513fe2983f5b Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:15:41 +0000 Subject: [PATCH 090/100] Update datashuttle/datashuttle_class.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/datashuttle_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 4b98b8fb4..9612e23e9 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -979,7 +979,7 @@ def _try_encrypt_rclone_config( ) -> None: """Try to encrypt the rclone config file. - If it fails, error and let the user the config file is unencrypted. + If it fails, error and let the user know the config file is unencrypted. """ try: self.encrypt_rclone_config() From 531eb022531b48f657936733e53ecfada784d8e1 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:16:02 +0000 Subject: [PATCH 091/100] Update datashuttle/configs/canonical_folders.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/configs/canonical_folders.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index e4d94d1d2..36d357365 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -108,12 +108,12 @@ def get_rclone_config_base_path() -> Path: as described here: https://rclone.org/docs/#config-string Because RClone's resolution process for where it stores its config files - is a little complex, in some rare cases the below may not match where - RClone stores its configs. This just means that local filesystem configs, + is a little complex, in some rare cases the path returned below may not match + where RClone actually stores its configs. In such cases, local filesystem configs, which are stored in the default `rclone.conf` file for backwards compatibility - reasons. and transfer configs which are stored in their own file at the - path returned from this function, are stored in a separate places, - which is not a huge deal. + reasons, and transfer configs, which are stored in their own file at the path + returned from this function, are stored in separate places. This is generally + not a significant issue. """ if platform.system() == "Windows": appdata_path = Path().home() / "AppData" / "Roaming" From 8f14c9e216c9ed0dc0a51543799ae5b113503bac Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:17:09 +0000 Subject: [PATCH 092/100] Update datashuttle/datashuttle_class.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/datashuttle_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 9612e23e9..cf689b125 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -991,7 +991,7 @@ def _try_encrypt_rclone_config( utils.log_and_raise_error( f"{str(e)}\n" f"Config encryption failed.\n" - f"Use encrypt_rclone_config()` to attempt to encrypt the file again " + f"Use `encrypt_rclone_config()` to attempt to encrypt the file again " f"(see full error message above).\n" f"IMPORTANT: The config at {config_path} is not currently encrypted.\n", RuntimeError, From 07bef3d27d2c100d1ad4f9435132e975899df9eb Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:17:46 +0000 Subject: [PATCH 093/100] Update datashuttle/configs/rclone_configs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/configs/rclone_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/configs/rclone_configs.py b/datashuttle/configs/rclone_configs.py index dfe41ba9f..c940774c0 100644 --- a/datashuttle/configs/rclone_configs.py +++ b/datashuttle/configs/rclone_configs.py @@ -95,7 +95,7 @@ def set_rclone_config_encryption_state(self, value: bool) -> None: def rclone_file_is_encrypted( self, ) -> bool: - """Return whether the config file associated with the current `connection_method`.""" + """Return whether the config file associated with the current `connection_method` is encrypted.""" assert rclone_encryption.connection_method_requires_encryption( self.datashuttle_configs["connection_method"] ) From e49b526a27acca269018a48a6874cd2938b3170d Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 16:22:26 +0000 Subject: [PATCH 094/100] Fix broken datashuttle version check. --- datashuttle/__init__.py | 4 ---- datashuttle/utils/rclone.py | 19 ++++--------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/datashuttle/__init__.py b/datashuttle/__init__.py index 3835203aa..557521387 100644 --- a/datashuttle/__init__.py +++ b/datashuttle/__init__.py @@ -9,7 +9,3 @@ except PackageNotFoundError: # package is not installed pass - - -def get_datashuttle_version(): - return __version__ diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d0871e87d..ad8af6f90 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -19,8 +19,6 @@ import subprocess import tempfile -from packaging import version - from datashuttle.configs import canonical_configs from datashuttle.utils import rclone_encryption, utils @@ -188,24 +186,15 @@ def run_function_that_requires_encrypted_rclone_config_access( In this case we need to set an environment variable to tell Rclone how to decrypt the config file (and remove the variable afterwards). """ - from datashuttle import get_datashuttle_version # avoid circular import - rclone_config_filepath = ( cfg.rclone.get_rclone_central_connection_config_filepath() ) if check_config_exists and not rclone_config_filepath.is_file(): - if version.parse(get_datashuttle_version()) <= version.parse("0.7.1"): - raise RuntimeError( - f"The way RClone configs are managed has changed since version v0.7.1\n" - f"Please set up the {cfg['connection_method']} connection again." - ) - else: - raise RuntimeError( - f"An unexpected error occurred. Could not find the rclone config " - f"file at: {rclone_config_filepath}\n" - f"Please set up the {cfg['connection_method']} connection again." - ) + raise RuntimeError( + f"The way RClone configs are managed has changed since version v0.7.1\n" + f"Please set up the {cfg['connection_method']} connection again." + ) is_encrypted = cfg.rclone.rclone_file_is_encrypted() From 4d5450e2b589343fc529e0be0724e2648e0ee918 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:23:34 +0000 Subject: [PATCH 095/100] Update datashuttle/utils/rclone_encryption.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/utils/rclone_encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/utils/rclone_encryption.py b/datashuttle/utils/rclone_encryption.py index 67d1ec50e..df1ad5448 100644 --- a/datashuttle/utils/rclone_encryption.py +++ b/datashuttle/utils/rclone_encryption.py @@ -125,7 +125,7 @@ def set_password_linux(cfg: Configs) -> None: utils.log_and_raise_error( f"\n--- STDOUT ---\n{output.stdout}" f"\n--- STDERR ---\n{output.stderr}" - "Could encrypt the password from the RClone config. See the error message above.", + "Could not encrypt the password from the RClone config. See the error message above.", RuntimeError, ) From 34981f586e3f095e264a623dcb49ce7f69cffc40 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:23:51 +0000 Subject: [PATCH 096/100] Update docs/source/pages/get_started/set-up-a-project.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/source/pages/get_started/set-up-a-project.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index 1bcd5b923..d7bcf8ee3 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -565,7 +565,7 @@ This file can include: By default, these are stored in your home directory which should be secure. However, for an additional layer of security, it is possible to encrypt the Rclone config file using the system credential manager of your operating system. This file will then be -non-readable for anyone who does not have access to your machine user account. Note that +unreadable for anyone who does not have access to your machine user account. Note that anyone with access to the machine user account will be able to decrypt the Rclone file. Despite this layer of security, it is not recommended to use datashuttle for remote connectivity on From 33ce2d7a85af79cf144ca2c59ab062cd757a31c5 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:24:06 +0000 Subject: [PATCH 097/100] Update datashuttle/tui/screens/setup_aws.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/tui/screens/setup_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 8df42cfaf..3c2a2b995 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -143,6 +143,6 @@ def set_rclone_encryption(self): self.stage = "finished" else: message = ( - f"The rclone_encryption set up failed. Exception: {output}" + f"The rclone encryption set up failed. Exception: {output}" ) self.query_one("#setup_aws_messagebox_message").update(message) From bf84a44385093f4a77e482844e2840915da3aa89 Mon Sep 17 00:00:00 2001 From: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:24:21 +0000 Subject: [PATCH 098/100] Update datashuttle/tui/screens/setup_aws.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- datashuttle/tui/screens/setup_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index 3c2a2b995..c5c09ec72 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -97,7 +97,7 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.stage = "use_secret_access_key" def use_secret_access_key_to_setup_aws_connection(self) -> None: - """Set up the AWS connection and failure. + """Set up the AWS connection and inform user of success or failure. If success, move onto the rclone_encryption screen. """ From 07a3c0e61fc92ea708ada2b0f4001f19422f2e09 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 16:40:18 +0000 Subject: [PATCH 099/100] Small fixes to sucess message. --- datashuttle/tui/screens/setup_aws.py | 8 +++----- tests/tests_transfers/aws/test_tui_setup_aws.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/datashuttle/tui/screens/setup_aws.py b/datashuttle/tui/screens/setup_aws.py index c5c09ec72..fd705aa7d 100644 --- a/datashuttle/tui/screens/setup_aws.py +++ b/datashuttle/tui/screens/setup_aws.py @@ -97,7 +97,7 @@ def prompt_user_for_aws_secret_access_key(self) -> None: self.stage = "use_secret_access_key" def use_secret_access_key_to_setup_aws_connection(self) -> None: - """Set up the AWS connection and inform user of success or failure. + """Set up the AWS connection and failure. If success, move onto the rclone_encryption screen. """ @@ -134,15 +134,13 @@ def set_rclone_encryption(self): success, output = self.interface.try_setup_rclone_encryption() if success: - message = ( - "The rclone_encryption was successfully set. Setup complete!" - ) + message = "The Rclone config file was successfully encrypted. Setup complete!" self.query_one("#setup_aws_messagebox_message").update(message) self.query_one("#setup_aws_ok_button").label = "Finish" self.query_one("#setup_aws_cancel_button").remove() self.stage = "finished" else: message = ( - f"The rclone encryption set up failed. Exception: {output}" + f"The rclone_encryption set up failed. Exception: {output}" ) self.query_one("#setup_aws_messagebox_message").update(message) diff --git a/tests/tests_transfers/aws/test_tui_setup_aws.py b/tests/tests_transfers/aws/test_tui_setup_aws.py index e184ee4ca..29039c77a 100644 --- a/tests/tests_transfers/aws/test_tui_setup_aws.py +++ b/tests/tests_transfers/aws/test_tui_setup_aws.py @@ -78,7 +78,7 @@ async def test_aws_connection_setup( await self.scroll_to_click_pause(pilot, "#setup_aws_ok_button") assert ( - "The rclone_encryption was successfully set. Setup complete!" + "The Rclone config file was successfully encrypted. Setup complete!" in pilot.app.screen.query_one( "#setup_aws_messagebox_message" ).renderable From f42d7b265d931ceed145ba3bc7a599b6417cda8f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Thu, 20 Nov 2025 17:06:10 +0000 Subject: [PATCH 100/100] Tidy up docs. --- .../pages/get_started/set-up-a-project.md | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/source/pages/get_started/set-up-a-project.md b/docs/source/pages/get_started/set-up-a-project.md index d7bcf8ee3..84a8da396 100644 --- a/docs/source/pages/get_started/set-up-a-project.md +++ b/docs/source/pages/get_started/set-up-a-project.md @@ -551,26 +551,24 @@ Running [](setup_aws_connection()) will require entering your (password-protection)= -# Password protecting your connection credentials +# Encrypting your connection credentials -Datashuttle uses [RClone](https://rclone.org/) for all data transfers by default. -RClone stores connection credentials (such as SSH keys or API tokens) in a local configuration file that, by default, is not encrypted. +Datashuttle uses [RClone](https://rclone.org/) for all data transfers. +RClone stores connection credentials in a +local configuration file that, by default, is not encrypted. This file can include: -- SSH connections: your private SSH key -- Google Drive connections: your OAuth access token and client secret -- Amazon S3 connections: your AWS Access Key ID and Secret Access Key +- **SSH:** your private SSH key +- **Google Drive:** your OAuth access token and client secret +- **Amazon S3:** your AWS Access Key ID and Secret Access Key -By default, these are stored in your home directory which should be secure. However, for an +These are stored in your home directory, which is expected to be secure. However, for an additional layer of security, it is possible to encrypt the Rclone config file using the system credential manager of your operating system. This file will then be unreadable for anyone who does not have access to your machine user account. Note that anyone with access to the machine user account will be able to decrypt the Rclone file. -Despite this layer of security, it is not recommended to use datashuttle for remote connectivity on -a machine to which you do not have secure access, even with user account encryption of the RClone config. - For details on setting up encryption, see the section below. On Windows, you will need to be running in PowerShell, and on Linux you will need `pass` package installed. @@ -578,9 +576,9 @@ need to be running in PowerShell, and on Linux you will need `pass` package inst :::{tab-item} Windows -On Windows, Datashuttle uses the PowerShell `PSCredential` system to encrypt the RClone config file. +On Windows, the PowerShell `PSCredential` system to encrypt the RClone config file. -- A random password is generated and stored as a `.clixml` credential file under a `credentials` folder in the project config location. +- A random password is generated and stored as a `.clixml` credential file. - The password can only be decrypted by the same Windows user account that created it. - The encryption and decryption process uses PowerShell, so PowerShell must be available (it will not work from `cmd.exe`). @@ -590,7 +588,7 @@ When encryption is enabled, RClone automatically retrieves the password from the :::{tab-item} macOS -On macOS, Datashuttle uses the built-in Keychain via the `security` command-line tool. +On macOS, the built-in Keychain via the `security` command-line tool is used. - A random password is generated using `openssl rand -base64 40`. - The password is securely stored in your login Keychain under the service name corresponding to your RClone config. @@ -603,11 +601,12 @@ Once approved, RClone will automatically retrieve the key when needed. :::{tab-item} Linux -1. Install `pass`: +On Linux, the `pass` package is used to manage the encryption. You can install it with: ```bash sudo apt install pass ``` -2. Initialize the password store with your GPG key: + +Next, you need to initialize the password store with your GPG key: ```bash pass init ``` @@ -626,7 +625,7 @@ Once initialized, Datashuttle will: ## Removing encryption -Encryption of the rclone config used for the central connection (either SSH, Google Drive or AWS) -can be removed with the following command: +Encryption of the Rclone config file used for the central connection +(either SSH, Google Drive or AWS) can be removed with the following command: [](remove_rclone_encryption())