Instead of downloading pregenerated backup ZIPs (which now require a …

…manual login they previously didn't), generate the backups, keeping the exact same ZIP layout, from the Todoist API, which still works correctly.

README.md

@@ -40,17 +40,13 @@ Early version - enough for basic use, but not tested under every possible scenar
 
 ## Usage examples
 
-Download latest backup from Todoist's servers, including attachments:
+Create a backup from Todoist's servers, including attachments:
 
-``python3 __main__.py download LATEST --with-attachments --token 0123456789abcdef``
-
-Download a specific backup from Todoist's servers, without including attachments:
-
-``python3 __main__.py download "2018-02-16 06:46" --token 0123456789abcdef``
+``python3 __main__.py download --with-attachments --token 0123456789abcdef``
 
 Download a specific backup from Todoist's servers, including attachments, and with tracing/progress info:
 
-``python3 __main__.py --verbose download "2018-02-16 06:46" --token 0123456789abcdef``
+``python3 __main__.py --verbose download --token 0123456789abcdef``
 
 List available backups:
 

experiment-webapp/index.html

@@ -7,7 +7,7 @@
         <script>
         function askCommandLine() {
             return prompt("Enter the command line args",
-                "todoist-offline-backup-dl --verbose download LATEST --token XXX");
+                "todoist-offline-backup-dl --verbose download --token XXX");
         }
 
         function hexEncode(str) {

todoist_full_offline_backup/backup_downloader.py

@@ -1,44 +1,35 @@
 #!/usr/bin/python3
 """ Class to download Todoist backup ZIPs using the Todoist API """
-import zipfile
-import io
+import datetime
 from .utils import sanitize_file_name
 
 class TodoistBackupDownloader:
     """ Class to download Todoist backup ZIPs using the Todoist API """
     __ZIP_FLAG_BITS_UTF8 = 0x800
 
-    def __init__(self, tracer, urldownloader):
+    def __init__(self, tracer, todoist_api):
         self.__tracer = tracer
-        self.__urldownloader = urldownloader
+        self.__todoist_api = todoist_api
 
-    def download(self, backup, vfs):
-        """ Downloads the specified backup to the specified folder """
-        self.__tracer.trace("Downloading backup with version '{}'".format(backup.version))
+    def download(self, vfs):
+        """ Generates a Todoist backup and saves it to the given VFS """
+        self.__tracer.trace("Generating backup from current Todoist status")
 
         # Sanitize the file name for platforms such as Windows,
         # which don't accept some characters in file names, such as a colon (:)
-        vfs.set_path_hint(sanitize_file_name("TodoistBackup_" + backup.version))
+        backup_version = datetime.datetime.utcnow().replace(microsecond=0).isoformat(' ')
+        vfs.set_path_hint(sanitize_file_name("TodoistBackup_" + backup_version))
 
         # Download the file
         if vfs.existed():
             self.__tracer.trace("File already downloaded... skipping")
             return
 
-        self.__tracer.trace("Downloading from {}...".format(
-            backup.url))
-        raw_zip_bytes = self.__urldownloader.get(backup.url)
-        with zipfile.ZipFile(io.BytesIO(raw_zip_bytes), "r") as zipf:
-            for info in zipf.infolist():
-                # Todoist backup ZIPs may contain filenames encoded in UTF-8, but they will not
-                # actually have the UTF-8 filename flag set in the ZIP file.
-                # This causes some ZIP parsers, such as Python's own parser, to consider the
-                # file names in the legacy CP-437 format.
-                # To fix this, let's re-encode the filenames in CP-437 to get the original
-                # bytes back, then properly decode them to UTF-8.
-                if info.flag_bits & self.__ZIP_FLAG_BITS_UTF8:
-                    encoding_file_name = info.filename
-                else:
-                    encoding_file_name = info.filename.encode('cp437').decode("utf-8")
+        self.__tracer.trace("Downloading project list from todoist API...")
+        projects = self.__todoist_api.get_projects()
 
-                vfs.write_file(encoding_file_name, zipf.read(info.filename))
+        for project in projects:
+            export_csv_file_name = "{} [{}].csv".format(
+                sanitize_file_name(project.name), project.identifier)
+            export_csv_file_content = self.__todoist_api.export_project_as_csv(project)
+            vfs.write_file(export_csv_file_name, export_csv_file_content)

todoist_full_offline_backup/controller.py

@@ -3,10 +3,6 @@
 
 from abc import ABCMeta, abstractmethod
 
-class BackupNotFoundException(Exception):
-    """ Thrown when the user requests a download of a backup that does not exist """
-    pass
-
 class ControllerDependencyInjector(object, metaclass=ABCMeta):
     """ Rudimentary dependency injection container for the controller """
 
@@ -17,18 +13,15 @@ def __init__(self, token, verbose):
     @property
     @abstractmethod
     def tracer(self):
-        """ Get an instance of the tracer """
-
-    @property
-    @abstractmethod
-    def todoist_api(self):
-        """ Gets an instance of the Todoist API """
+        """ Gets an instance of the debug tracer """
 
     @property
     @abstractmethod
     def backup_downloader(self):
         """ Gets an instance of the Todoist backup downloader """
 
+    @property
+    @abstractmethod
     def backup_attachments_downloader(self):
         """ Gets an instance of the Todoist backup attachment downloader """
 
@@ -38,38 +31,8 @@ class Controller:
     def __init__(self, dependencies):
         self.__dependencies = dependencies
 
-    def get_backups(self):
-        """ Gets the list of backups along with the latest backup """
-        backups = self.__dependencies.todoist_api.get_backups()
-        return (backups, self.__get_latest(backups))
-
-    def download_latest(self, vfs, with_attachments):
-        """ Downloads the latest (i.e. most recent) Todoist backup ZIP """
-        self.__dependencies.tracer.trace("Fetching backup list to find the latest backup...")
-        backups = self.__dependencies.todoist_api.get_backups()
-        backup = self.__get_latest(backups)
-        if backup is None:
-            raise BackupNotFoundException()
-        self.__dependencies.backup_downloader.download(backup, vfs)
-        if with_attachments:
-            self.__dependencies.backup_attachments_downloader.download_attachments(vfs)
-
-    def download_version(self, version, vfs, with_attachments):
-        """ Downloads the specified Todoist backup ZIP given by its version string """
-        self.__dependencies.tracer.trace(
-            "Fetching backup list to find the backup '{}'...".format(version))
-        backups = self.__dependencies.todoist_api.get_backups()
-        backup = self.__find_version(backups, version)
-        if backup is None:
-            raise BackupNotFoundException()
-        self.__dependencies.backup_downloader.download(backup, vfs)
+    def download(self, vfs, with_attachments):
+        """ Generates a Todoist backup ZIP from the current Todoist items """
+        self.__dependencies.backup_downloader.download(vfs)
         if with_attachments:
             self.__dependencies.backup_attachments_downloader.download_attachments(vfs)
-
-    @staticmethod
-    def __get_latest(backups):
-        return max(backups, key=lambda x: x.version_date, default=None)
-
-    @staticmethod
-    def __find_version(backups, version):
-        return next((x for x in backups if x.version == version), None)

todoist_full_offline_backup/frontend.py

@@ -21,24 +21,16 @@ def __add_token_group(parser):
                                  help="file containing the todoist API token")
 
     def __parse_command_line_args(self, prog, arguments):
-        example1_str = "Example: {} download LATEST --token 0123456789abcdef".format(prog)
-        example2_str = "         {} --verbose list --token 0123456789abcdef".format(prog)
+        example1_str = "Example: {} download --token 0123456789abcdef".format(prog)
         parser = argparse.ArgumentParser(prog=prog, formatter_class=argparse.RawTextHelpFormatter,
-                                         epilog=example1_str + '\r\n' + example2_str)
+                                         epilog=example1_str)
         parser.add_argument("--verbose", action="store_true", help="print details to console")
         subparsers = parser.add_subparsers(dest='action')
         subparsers.required = True
 
-        # create the parser for the "list" command
-        parser_list = subparsers.add_parser('list', help='list available backups')
-        parser_list.set_defaults(func=self.handle_list)
-        self.__add_token_group(parser_list)
-
         # create the parser for the "download" command
         parser_download = subparsers.add_parser('download', help='download specified backup')
         parser_download.set_defaults(func=self.handle_download)
-        parser_download.add_argument("version", type=str, metavar="VERSIONSPEC|LATEST",
-                                     help="backup version to download, or LATEST")
         parser_download.add_argument("--with-attachments", action="store_true",
                                      help="download attachments and attach to the backup file")
         parser_download.add_argument("--output-file", type=str,
@@ -59,20 +51,6 @@ def __get_token(args):
 
         return args.token
 
-    def handle_list(self, args):
-        """ Handles the list subparser with the specified command line arguments """
-        # Configure controller
-        token = self.__get_token(args)
-        dependencies = self.__controller_dependencies_factory(token, args.verbose)
-        controller = self.__controller_factory(dependencies)
-
-        # Execute requested action
-        (backups, latest_backup) = controller.get_backups()
-        print("{:<30} | {}".format("VERSION", "URL"))
-        for backup in backups:
-            is_latest_str = " (LATEST)" if backup == latest_backup else ""
-            print("{:<30} | {}".format(backup.version + is_latest_str, backup.url))
-
     def handle_download(self, args):
         """ Handles the download subparser with the specified command line arguments """
 
@@ -84,8 +62,4 @@ def handle_download(self, args):
         # Setup zip virtual fs
         with ZipVirtualFs(args.output_file) as zipvfs:
             # Execute requested action
-            if args.version == 'LATEST':
-                controller.download_latest(zipvfs, with_attachments=args.with_attachments)
-            else:
-                controller.download_version(args.version, zipvfs,
-                                            with_attachments=args.with_attachments)
+            controller.download(zipvfs, with_attachments=args.with_attachments)

todoist_full_offline_backup/runtime.py

@@ -15,19 +15,15 @@ def __init__(self, token, verbose):
         super(RuntimeControllerDependencyInjector, self).__init__(token, verbose)
         self.__tracer = ConsoleTracer() if verbose else NullTracer()
         urldownloader = URLLibURLDownloader()
-        self.__todoist_api = TodoistApi(token, self.__tracer, urldownloader)
-        self.__backup_downloader = TodoistBackupDownloader(self.__tracer, urldownloader)
+        todoist_api = TodoistApi(token, self.__tracer, urldownloader)
+        self.__backup_downloader = TodoistBackupDownloader(self.__tracer, todoist_api)
         self.__backup_attachments_downloader = TodoistBackupAttachmentsDownloader(
             self.__tracer, urldownloader)
 
     @property
     def tracer(self):
         return self.__tracer
 
-    @property
-    def todoist_api(self):
-        return self.__todoist_api
-
     @property
     def backup_downloader(self):
         return self.__backup_downloader

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147711.csv

@@ -0,0 +1 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147712.csv

@@ -0,0 +1 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147713.csv

@@ -0,0 +1 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147714.csv

@@ -0,0 +1 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147715.csv

@@ -0,0 +1 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE

todoist_full_offline_backup/tests/integration_files/sources/Project_2181147955.csv

@@ -0,0 +1,4 @@
+TYPE,CONTENT,PRIORITY,INDENT,AUTHOR,RESPONSIBLE,DATE,DATE_LANG,TIMEZONE
+task,this task has a 🐛 in its name!!!,4,1,test_integration_test (16542905),,,en,Europe/Madrid
+note," [[file {""file_size"":28,""file_type"":""text\/plain"",""file_name"":""bug.txt"",""upload_state"":""completed"",""file_url"":""https:\/\/d1x0mwiac2rqwt.cloudfront.net\/g75-kL8pwVYNObSczLnVXe4FIyJd8YQL6b8yCilGyix09bMdJmxbtrGMW9jIeIwJ\/by\/16542905\/as\/bug.txt"",""resource_type"":""file""}]]",,,test_integration_test (16542905),,,,
+,,,,,,,,

todoist_full_offline_backup/tests/integration_files/sources/project_list.json

@@ -0,0 +1,91 @@
+{
+    "projects" : [
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147955,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "🦋 butterflies list",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147714,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Errands",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147709,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Inbox",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147715,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Movies to watch",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147711,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Personal",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147712,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Shopping",
+            "is_deleted" : 0,
+            "indent" : 1
+        },
+        {
+            "is_archived" : 0,
+            "color" : 7,
+            "shared" : false,
+            "inbox_project" : true,
+            "id" : 2181147713,
+            "collapsed" : 0,
+            "item_order" : 0,
+            "name" : "Work",
+            "is_deleted" : 0,
+            "indent" : 1
+        }
+    ],
+    "full_sync" : true,
+    "temp_id_mapping" : {},
+    "sync_token" : "aLGJg_2qwBE_kE3j9_Gn6uoKQtvQeyjm7UEz_aVwF8KdriDxw7e_InFZK61h"
+ }

todoist_full_offline_backup/tests/test_backup_attachments_downloader.py

@@ -37,7 +37,7 @@ def test_on_simple_download_downloads_attachments(self):
         writer.writerow(["task", "This is a random task", "4"])
         writer.writerow(["note", " [[file {}]]".format(json.dumps({
             "file_type": "image/jpg",
-            "file_name": "this/is/an/image.jpg",
+            "file_name": "this_is_an_image.jpg",
             "file_url": self._TEST_FILE_JPG_URL
         })), "test"])