More fixes to migrate to Todoist v9 Sync API.

This prorgam was failing again, and it appears that this time it is because
there are more changes in the Todoist v9 Sync API that it initially appeared:
* GET requests are now POST
* data must be passed as FormData instead of QueryString
* The token must be passed in 'Authorization: Bearer ...' instead of data.

For some reason this wasn't necessary 1 week ago, I guess they still kept some
v8 API code which is now removed or something. Anyway, use the API properly.

Author: joanbm

full_offline_backup_for_todoist/tests/test_integration.py

@@ -29,25 +29,25 @@ def setUp(self):
         # Set up the fake HTTP server with local responses
         # pylint: disable=line-too-long
         route_responses = {
-            "/https://api.todoist.com/sync/v9/sync?token=mysecrettoken&sync_token=%2A&resource_types=%5B%22projects%22%5D":
+            ("POST", "/https://api.todoist.com/sync/v9/sync", b"sync_token=%2A&resource_types=%5B%22projects%22%5D", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/project_list.json")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147955":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147955", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147955.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147714":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147714", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147714.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147709":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147709", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147709.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147715":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147715", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147715.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147711":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147711", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147711.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147712":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147712", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147712.csv")).read_bytes(),
-            "/https://api.todoist.com/sync/v9/templates/export_as_file?token=mysecrettoken&project_id=2181147713":
+            ("POST", "/https://api.todoist.com/sync/v9/templates/export_as_file", b"project_id=2181147713", 'mysecrettoken'):
                 Path(self.__get_test_file("sources/Project_2181147713.csv")).read_bytes(),
-            "/https://d1x0mwiac2rqwt.cloudfront.net/g75-kL8pwVYNObSczLnVXe4FIyJd8YQL6b8yCilGyix09bMdJmxbtrGMW9jIeIwJ/by/16542905/as/bug.txt":
+            ("GET, /https://d1x0mwiac2rqwt.cloudfront.net/g75-kL8pwVYNObSczLnVXe4FIyJd8YQL6b8yCilGyix09bMdJmxbtrGMW9jIeIwJ/by/16542905/as/bug.txt", None, None):
                 Path(self.__get_test_file("sources/bug.txt")).read_bytes(),
-            "/https://d1x0mwiac2rqwt.cloudfront.net/s0snyb7n9tJXYijOK2LV6hjVar4YUkwYbHv3PBFYM-N4nJEtujC046OlEdZpKfZm/by/16542905/as/sample_image.png":
+            ("GET, /https://d1x0mwiac2rqwt.cloudfront.net/s0snyb7n9tJXYijOK2LV6hjVar4YUkwYbHv3PBFYM-N4nJEtujC046OlEdZpKfZm/by/16542905/as/sample_image.png", None, None):
                 Path(self.__get_test_file("sources/sample_image.png")).read_bytes(),
         }
 
@@ -57,12 +57,12 @@ def tearDown(self):
         """ Destroys the sample HTTP server for the test """
         self.__httpd.shutdown()
 
-    def __opener_open_redirect_to_local(self, original_self, url):
+    def __opener_open_redirect_to_local(self, original_self, url, data):
         """ Replaces the OpenerDirector.open function of URLLib, in order to redirect all requests
             to a local server.
             This way, we are still able to do the integration test with actual HTTP requests,
             though being handled by a local test HTTP server """
-        return self.__original_opener_open(original_self, "http://127.0.0.1:33327/" + url)
+        return self.__original_opener_open(original_self, "http://127.0.0.1:33327/" + url, data)
 
     @staticmethod
     def __get_test_file(subpath):

full_offline_backup_for_todoist/tests/test_todoist_api.py

@@ -60,8 +60,8 @@ def test_on_call_get_projects_with_token_calls_download_with_token():
         todoist_api.get_projects()
 
         # Assert
+        mock_urldownloader.set_bearer_token.assert_called_with('FAKE TOKEN')
         mock_urldownloader.get.assert_called_with(ANY, {
-            'token': 'FAKE TOKEN',
             'sync_token': '*',
             'resource_types': '["projects"]'
         })

full_offline_backup_for_todoist/tests/test_url_downloader.py

@@ -14,8 +14,10 @@ def setUp(self):
 
         # Set up a quick and dirty HTTP server
         route_responses = {
-            "/sample.txt": "this is a sample".encode(),
-            "/sample.txt?param=value": "this is a sample with a parameter".encode(),
+            ("GET", "/sample.txt", None, None):
+                "this is a sample".encode(),
+            ("POST", "/sample.txt", b"param=value", None):
+                "this is a sample with a parameter".encode(),
         }
 
         self.__httpd = TestStaticHTTPServer(("127.0.0.1", 33327), route_responses)

full_offline_backup_for_todoist/tests/test_util_static_http_request_handler.py

@@ -29,20 +29,41 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
             def log_message(self, format, *args): # pylint: disable=redefined-builtin
                 """ Disables console output for the HTTP Request Handler """
 
-            def do_GET(self): # pylint: disable=invalid-name
-                """ Handles a request using the defined static mapping """
+            def __find_response_for_request_key(self, request_key):
+                authorization_header = self.headers.get('Authorization')
+                if authorization_header is not None and authorization_header.startswith("Bearer "):
+                    token = authorization_header[len("Bearer "):]
+                    key_with_auth = request_key + (token,)
+                    if key_with_auth in route_responses:
+                        return route_responses[key_with_auth]
+
+                key_without_auth = request_key + (None,)
+                return route_responses.get(key_without_auth)
+
+            def __handle_request(self, request_key):
                 if flaky and self.path not in handled:
                     handled.add(self.path)
                     self.send_response(503)
                     self.end_headers()
                     return
 
-                self.send_response(200 if self.path in route_responses else 404)
+                response = self.__find_response_for_request_key(request_key)
+                self.send_response(200 if response else 404)
 
                 self.send_header('Content-type', 'text/plain')
                 self.end_headers()
 
-                if self.path in route_responses:
-                    self.wfile.write(route_responses[self.path])
+                if response:
+                    self.wfile.write(response)
+
+            def do_GET(self): # pylint: disable=invalid-name
+                """ Handles a GET request using the defined static mapping """
+                self.__handle_request(('GET', self.path, None))
+
+            def do_POST(self): # pylint: disable=invalid-name
+                """ Handles a POST request using the defined static mapping """
+                body_length = int(self.headers.get('Content-Length'))
+                body = self.rfile.read(body_length)
+                self.__handle_request(('POST', self.path, body))
 
         return TestHTTPRequestHandler

full_offline_backup_for_todoist/todoist_api.py

@@ -23,21 +23,19 @@ class TodoistApi:
     __SYNC_ENDPOINT = __BASE_URL + "/sync"
     __EXPORT_PROJECT_AS_CSV_FILE_ENDPOINT = __BASE_URL + "/templates/export_as_file"
 
-    __api_token: str
     __tracer: Tracer
     __urldownloader: URLDownloader
 
     def __init__(self, api_token: str, tracer: Tracer, urldownloader: URLDownloader):
-        self.__api_token = api_token
         self.__tracer = tracer
         self.__urldownloader = urldownloader
+        self.__urldownloader.set_bearer_token(api_token)
 
     def get_projects(self) -> List[TodoistProjectInfo]:
         """ Obtains the list of all projects from the Todoist API """
         self.__tracer.trace("Fetching projects using the Todoist API...")
         project_list_json = self.__urldownloader.get(
             self.__SYNC_ENDPOINT, {
-                "token": self.__api_token,
                 "sync_token": '*',
                 "resource_types": '["projects"]',
             })
@@ -55,6 +53,5 @@ def export_project_as_csv(self, project: TodoistProjectInfo) -> bytes:
 
         return self.__urldownloader.get(
             self.__EXPORT_PROJECT_AS_CSV_FILE_ENDPOINT, {
-                "token": self.__api_token,
                 "project_id": project.identifier
             })

full_offline_backup_for_todoist/url_downloader.py

@@ -14,48 +14,53 @@ class URLDownloader(metaclass=ABCMeta):
     """ Implementation of a class to download the contents of an URL """
 
     _tracer: Tracer
+    __bearer_token: Optional[str]
 
     def __init__(self, tracer: Tracer):
         self._tracer = tracer
+        self.__bearer_token = None
 
-    def _download(self, opener: urllib.request.OpenerDirector, url: str) -> bytes:
+    def _download(self, opener: urllib.request.OpenerDirector, url: str,
+                  data: Optional[Dict[str, str]]=None) -> bytes:
         """ Downloads the specified URL as bytes using the specified opener """
-        with opener.open(url) as url_handle:
+        encoded_data = urllib.parse.urlencode(data).encode() if data else None
+        with opener.open(url, encoded_data) as url_handle:
             return cast(bytes, url_handle.read())
 
-    def _download_with_retry(self, opener: urllib.request.OpenerDirector, url: str) -> bytes:
+    def _download_with_retry(self, opener: urllib.request.OpenerDirector, url: str,
+                             data: Optional[Dict[str, str]]=None) -> bytes:
         """ Downloads the specified URL as bytes using the specified opener, retrying on failure """
         for i in range(NUM_RETRIES):
             try:
-                return self._download(opener, url)
+                return self._download(opener, url, data)
             except urllib.error.URLError as exception:
                 self._tracer.trace(f"Got exception: {exception}, retrying...")
                 time.sleep(3**i)
 
-        return self._download(opener, url)
+        return self._download(opener, url, data)
+
+    def set_bearer_token(self, bearer_token: Optional[str]) -> None:
+        """ Sets the value of the 'Authorization: Bearer XXX' HTTP header """
+        self.__bearer_token = bearer_token
 
     @abstractmethod
     def get(self, url: str, data: Optional[Dict[str, str]]=None) -> bytes:
         """ Download the contents of the specified URL with a GET request.
             You can specify any additional data parameters to pass to the destination. """
 
-    @staticmethod
     def _build_opener_with_app_useragent(
-        *handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector:
+        self, *handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector:
         opener = urllib.request.build_opener(*handlers)
-        opener.addheaders = [('User-agent', 'full-offline-backup-for-todoist')]
+        opener.addheaders = ([('User-agent', 'full-offline-backup-for-todoist')] +
+            ([('Authorization', 'Bearer ' + self.__bearer_token)] if self.__bearer_token else []))
         return opener
 
 class URLLibURLDownloader(URLDownloader):
     """ Implementation of a class to download the contents of an URL through URLLib """
 
     def get(self, url: str, data: Optional[Dict[str, str]]=None) -> bytes:
-        real_url = url
-        if data:
-            real_url += "?" + urllib.parse.urlencode(data)
-
         opener = self._build_opener_with_app_useragent()
-        return self._download_with_retry(opener, real_url)
+        return self._download_with_retry(opener, url, data)
 
 class TodoistAuthURLDownloader(URLDownloader):
     """ Implementation of a class to download the contents of an URL through URLLib,
@@ -110,8 +115,4 @@ def get(self, url: str, data: Optional[Dict[str, str]]=None) -> bytes:
 
             self._tracer.trace("Auth completed")
 
-        real_url = url
-        if data:
-            real_url += "?" + urllib.parse.urlencode(data)
-
-        return self._download_with_retry(self.__opener, real_url)
+        return self._download_with_retry(self.__opener, url, data)