Implemented --all for backup/restore, #33

Author: RhetTbull

README.md

@@ -31,8 +31,8 @@ If you only care about the command line tool, I recommend installing with [pipx]
 The command line tool can also be run via `python -m osxmetadata`.  Running it with no arguments or with --help option will print a help message:
 
 ```
-Usage: osxmetadata [OPTIONS] FILE
-  
+Usage: python -m osxmetadata [OPTIONS] FILE
+
   Read/write metadata from file(s).
 
 Options:
@@ -64,10 +64,16 @@ Options:
   -B, --backup                    Backup FILE attributes.  Backup file
                                   '.osxmetadata.json' will be created in same
                                   folder as FILE. Only backs up attributes
-                                  known to osxmetadata.
+                                  known to osxmetadata unless used with --all.
   -R, --restore                   Restore FILE attributes from backup file.
                                   Restore will look for backup file
                                   '.osxmetadata.json' in same folder as FILE.
+                                  Only restores attributes known to
+                                  osxmetadata unless used with --all.
+  -A, --all                       Process all extended attributes including
+                                  those not known to osxmetadata. Use with
+                                  --backup/--restore to backup/restore all
+                                  extended attributes.
   -V, --verbose                   Print verbose output.
   -f, --copyfrom SOURCE_FILE      Copy attributes from file SOURCE_FILE.
   --files-only                    Do not apply metadata commands to

osxmetadata/__main__.py

@@ -48,7 +48,7 @@ def __init__(self, debug=False, files=None):
 
 
 class MyClickCommand(click.Command):
-    """ Custom click.Command that overrides get_help() to show additional info """
+    """Custom click.Command that overrides get_help() to show additional info"""
 
     def get_help(self, ctx):
         help_text = super().get_help(ctx)
@@ -236,7 +236,7 @@ def get_help(self, ctx):
     "-B",
     help="Backup FILE attributes.  "
     "Backup file '.osxmetadata.json' will be created in same folder as FILE. "
-    "Only backs up attributes known to osxmetadata.",
+    "Only backs up attributes known to osxmetadata unless used with --all.",
     is_flag=True,
     required=False,
     default=False,
@@ -245,11 +245,20 @@ def get_help(self, ctx):
     "--restore",
     "-R",
     help="Restore FILE attributes from backup file.  "
-    "Restore will look for backup file '.osxmetadata.json' in same folder as FILE.",
+    "Restore will look for backup file '.osxmetadata.json' in same folder as FILE. "
+    "Only restores attributes known to osxmetadata unless used with --all.",
     is_flag=True,
     required=False,
     default=False,
 )
+ALL_OPTION = click.option(
+    "--all",
+    "-A",
+    "all_",
+    is_flag=True,
+    help="Process all extended attributes including those not known to osxmetadata. "
+    "Use with --backup/--restore to backup/restore all extended attributes. ",
+)
 VERBOSE_OPTION = click.option(
     "--verbose",
     "-V",
@@ -307,6 +316,7 @@ def get_help(self, ctx):
 @MIRROR_OPTION
 @BACKUP_OPTION
 @RESTORE_OPTION
+@ALL_OPTION
 @VERBOSE_OPTION
 @COPY_FROM_OPTION
 @FILES_ONLY_OPTION
@@ -330,12 +340,13 @@ def cli(
     mirror,
     backup,
     restore,
+    all_,
     verbose,
     copyfrom,
     files_only,
     pattern,
 ):
-    """ Read/write metadata from file(s). """
+    """Read/write metadata from file(s)."""
 
     if help_:
         click.echo_via_pager(ctx.get_help())
@@ -432,6 +443,7 @@ def cli(
                 restore,
                 walk,
                 files_only,
+                all_,
             )
 
         if walk and os.path.isdir(filename):
@@ -468,6 +480,7 @@ def cli(
                     restore,
                     walk,
                     files_only,
+                    all_,
                 )
 
 
@@ -490,10 +503,11 @@ def process_files(
     restore,
     walk,
     files_only,
+    all_,
 ):
-    """ process list of files, calls process_single_file to process each file
-        options processed in this order: wipe, copyfrom, clear, set, append, remove, mirror, get, list
-        Note: expects all attributes passed in parameters to be validated as valid attributes 
+    """process list of files, calls process_single_file to process each file
+    options processed in this order: wipe, copyfrom, clear, set, append, remove, mirror, get, list
+    Note: expects all attributes passed in parameters to be validated as valid attributes
     """
     for filename in files:
         fpath = pathlib.Path(filename).resolve()
@@ -514,7 +528,7 @@ def process_files(
                 if verbose:
                     click.echo(f"  Restoring attribute data for {fpath}")
                 md = osxmetadata.OSXMetaData(fpath)
-                md._restore_attributes(attr_dict)
+                md._restore_attributes(attr_dict, all_=all_)
             except FileNotFoundError:
                 click.echo(
                     f"Missing backup file {backup_file} for {fpath}, skipping restore",
@@ -547,12 +561,9 @@ def process_files(
             if verbose:
                 click.echo(f"  Backing up attribute data for {fpath}")
             # load the file if it exists, merge new data, then write out the file again
-            if backup_file.is_file():
-                backup_data = load_backup_file(backup_file)
-            else:
-                backup_data = {}
-            json_dict = osxmetadata.OSXMetaData(fpath).asdict()
-            backup_data[pathlib.Path(fpath).name] = json_dict
+            backup_data = load_backup_file(backup_file) if backup_file.is_file() else {}
+            backup_dict = osxmetadata.OSXMetaData(fpath).asdict(all_=all_)
+            backup_data[pathlib.Path(fpath).name] = backup_dict
             write_backup_file(backup_file, backup_data)
 
 
@@ -572,9 +583,9 @@ def process_single_file(
     verbose,
     copyfrom,
 ):
-    """ process a single file to apply the options 
-        options processed in this order: wipe, copyfrom, clear, set, append, remove, mirror, get, list
-        Note: expects all attributes passed in parameters to be validated as valid attributes """
+    """process a single file to apply the options
+    options processed in this order: wipe, copyfrom, clear, set, append, remove, mirror, get, list
+    Note: expects all attributes passed in parameters to be validated as valid attributes"""
 
     md = osxmetadata.OSXMetaData(fpath)
 

osxmetadata/_version.py

@@ -1 +1 @@
-__version__ = "0.99.14"
+__version__ = "0.99.15"

osxmetadata/osxmetadata.py

@@ -9,6 +9,7 @@
 import pathlib
 import plistlib
 import sys
+import base64
 
 # plistlib creates constants at runtime which causes pylint to complain
 from plistlib import FMT_BINARY  # pylint: disable=E0611
@@ -161,10 +162,18 @@ def tz_aware(self, tz_flag):
         tz_flag: bool"""
         self._tz_aware = tz_flag
 
-    def asdict(self):
-        """Return dict with all attributes for this file"""
+    def asdict(self, all_=False, encode=True):
+        """Return dict with all attributes for this file
 
-        attribute_list = self.list_metadata()
+        Args:
+            all_: bool, if True, returns all attributes including those that osxmetadata knows nothing about
+            encode: bool, if True, encodes values for unknown attributes with base64, otherwise leaves the values as raw bytes
+
+        Returns:
+            dict with attributes for this file
+        """
+
+        attribute_list = self._list_attributes() if all_ else self.list_metadata()
         dict_data = {
             "_version": __version__,
             "_filepath": self._posix_name,
@@ -194,20 +203,37 @@ def asdict(self):
                     # get raw value
                     dict_data[attribute.constant] = self.get_attribute(attribute.name)
             except KeyError:
-                # unknown attribute, ignore it
-                pass
+                # an attribute osxmetadata doesn't know about
+                if all_:
+                    try:
+                        value = self._attrs[attr]
+                        # convert value to base64 encoded ascii
+                        if encode:
+                            value = base64.b64encode(value).decode("ascii")
+                        dict_data[attr] = value
+                    except KeyError as e:
+                        # value disappeared between call to _list_attributes and now
+                        pass
         return dict_data
 
-    def to_json(self):
-        """Returns a string in JSON format for all attributes in this file"""
-        dict_data = self.asdict()
+    def to_json(self, all_=False):
+        """Returns a string in JSON format for all attributes in this file
+    
+        Args:
+            all_: bool; if True, also restores attributes not known to osxmetadata (generated with asdict(all_=True, encode=True) )
+        """
+        dict_data = self.asdict(all_=all_)
         return json.dumps(dict_data)
 
-    def _restore_attributes(self, attr_dict):
+    def _restore_attributes(self, attr_dict, all_=False):
         """restore attributes from attr_dict
         for each attribute in attr_dict, will set the attribute
         will not clear/erase any attributes on file that are not in attr_dict
-        attr_dict: an attribute dict as produced by OSXMetaData.asdict()"""
+
+        Args:
+            attr_dict: an attribute dict as produced by OSXMetaData.asdict()
+            all_: bool; if True, also restores attributes not known to osxmetadata (generated with asdict(all_=True, encode=True) )
+        """
 
         for key, val in attr_dict.items():
             if key.startswith("_"):
@@ -226,8 +252,10 @@ def _restore_attributes(self, attr_dict):
                             f"expected list for attribute {key} but got {type(val)}"
                         )
                     self.set_attribute(key, Tag(val[0], val[1]))
-                else:
+                elif key in ATTRIBUTES:
                     self.set_attribute(key, val)
+                elif all_:
+                    self._attrs.set(key, base64.b64decode(val))
             except Exception as e:
                 logging.warning(
                     f"Unable to restore attribute {key} for {self._fname}: {e}"
@@ -474,14 +502,14 @@ def clear_attribute(self, attribute_name):
             pass
 
     def _list_attributes(self):
-        """list the attributes set on the file"""
+        """list all the attributes set on the file"""
         return self._attrs.list()
 
     def list_metadata(self):
         """list the Apple metadata attributes set on the file:
         e.g. those in com.apple.metadata namespace"""
         # also lists com.osxmetadata.test used for debugging
-        mdlist = self._attrs.list()
+        mdlist = self._list_attributes()
         mdlist = [
             md
             for md in mdlist

tests/test_cli.py

@@ -48,7 +48,7 @@
 
 
 def create_file(filepath):
-    """ create an empty file at filepath """
+    """create an empty file at filepath"""
     fd = open(filepath, "w+")
     fd.close()
 
@@ -85,8 +85,8 @@ def temp_dir():
 
 
 def parse_cli_output(output):
-    """ helper for testing
-        parse the CLI --list output and return value of all set attributes as dict """
+    """helper for testing
+    parse the CLI --list output and return value of all set attributes as dict"""
     import re
 
     results = {}
@@ -412,6 +412,64 @@ def test_cli_backup_restore_2(temp_file):
     assert meta.keywords == ["FooBar"]
 
 
+def test_cli_backup_restore_all(temp_file):
+    """Test --backup/--restore with --all"""
+    import pathlib
+    from osxmetadata import OSXMetaData, ATTRIBUTES, Tag
+    from osxmetadata.constants import _BACKUP_FILENAME
+    from osxmetadata.__main__ import cli
+
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "--set",
+            "tags",
+            "Foo",
+            "--set",
+            "tags",
+            "Bar",
+            "--set",
+            "comment",
+            "Hello World!",
+            "--list",
+            temp_file,
+        ],
+    )
+    assert result.exit_code == 0
+    meta = OSXMetaData(temp_file)
+    # set a value osxmetadata doesn't know about
+    meta._attrs.set("com.foo.bar", b"FOOBAR")
+
+    # backup
+    result = runner.invoke(cli, ["--backup", "--all", temp_file])
+    assert result.exit_code == 0
+
+    # clear the attributes to see if they can be restored
+    meta.clear_attribute("tags")
+    meta.clear_attribute("comment")
+    meta._attrs.remove("com.foo.bar")
+    assert meta.tags == []
+    assert meta.comment is None
+    assert "com.foo.bar" not in meta._list_attributes()
+
+    # first run restore without --all
+    result = runner.invoke(cli, ["--restore", temp_file])
+    assert result.exit_code == 0
+    assert meta.tags == [Tag("Foo"), Tag("Bar")]
+    assert meta.comment == "Hello World!"
+
+    with pytest.raises(KeyError):
+        assert meta._attrs["com.foo.bar"] == b"FOOBAR"
+
+    # next run restore with --all
+    result = runner.invoke(cli, ["--restore", "--all", temp_file])
+    assert result.exit_code == 0
+    assert meta.tags == [Tag("Foo"), Tag("Bar")]
+    assert meta.comment == "Hello World!"
+    assert meta._attrs["com.foo.bar"] == b"FOOBAR"
+
+
 def test_cli_mirror(temp_file):
     import datetime
     from osxmetadata import OSXMetaData, Tag
@@ -794,7 +852,7 @@ def test_cli_downloadeddate(temp_file):
 
 
 def test_cli_walk(temp_dir):
-    """ test --walk """
+    """test --walk"""
     import os
     import pathlib
     from osxmetadata import OSXMetaData, Tag
@@ -818,7 +876,7 @@ def test_cli_walk(temp_dir):
 
 
 def test_cli_walk_files_only(temp_dir):
-    """ test --walk with --files-only """
+    """test --walk with --files-only"""
     import os
     import pathlib
     from osxmetadata import OSXMetaData, Tag
@@ -844,7 +902,7 @@ def test_cli_walk_files_only(temp_dir):
 
 
 def test_cli_walk_pattern(temp_dir):
-    """ test --walk with --pattern """
+    """test --walk with --pattern"""
     import os
     import pathlib
     from osxmetadata import OSXMetaData, Tag
@@ -874,7 +932,7 @@ def test_cli_walk_pattern(temp_dir):
 
 
 def test_cli_walk_pattern_2(temp_dir):
-    """ test --walk with more than one --pattern """
+    """test --walk with more than one --pattern"""
     import os
     import pathlib
     from osxmetadata import OSXMetaData, Tag
@@ -919,7 +977,7 @@ def test_cli_walk_pattern_2(temp_dir):
 
 
 def test_cli_files_only(temp_dir):
-    """ test --files-only without --walk """
+    """test --files-only without --walk"""
     import glob
     import os
     import pathlib