configurations

Author: cmungall

.gitignore

@@ -1,3 +1,5 @@
+\#*\#
+.#*
 env/
 py*/
 __pycache__
@@ -6,6 +8,7 @@ __pycache__
 tests/logs/*.log
 build
 dist
+.ipynb_checkpoints/
 
 /build
 /dist

EXAMPLE.ipynb

@@ -129,3 +129,7 @@ Documentation coming soon: see test folder for now
 
 ### Python
 
+### csvjson
+
+https://csvjson.com/json2csv
+

README.md

@@ -0,0 +1,115 @@
+{
+   "all_book_series" : [
+      {
+         "books" : [
+            {
+               "price" : 5.99,
+               "id" : "S001.1",
+               "summary" : "Hobbits",
+               "name" : "Fellowship of the Ring"
+            },
+            {
+               "price" : 5.99,
+               "summary" : "More hobbits",
+               "id" : "S001.2",
+               "name" : "The Two Towers"
+            },
+            {
+               "price" : 6.99,
+               "id" : "S001.3",
+               "summary" : "Yet more hobbits",
+               "name" : "Return of the King"
+            }
+         ],
+         "genres" : [
+            "fantasy"
+         ],
+         "id" : "S001",
+         "name" : "Lord of the Rings",
+         "creator" : {
+            "from_country" : "England",
+            "name" : "JRR Tolkein"
+         }
+      },
+      {
+         "name" : "The Culture Series",
+         "creator" : {
+            "name" : "Ian M Banks",
+            "from_country" : "Scotland"
+         },
+         "id" : "S002",
+         "genres" : [
+            "scifi"
+         ],
+         "books" : [
+            {
+               "price" : 5.99,
+               "id" : "S002.1",
+               "name" : "Consider Phlebas"
+            },
+            {
+               "price" : 5.99,
+               "name" : "Player of Games",
+               "id" : "S002.2"
+            }
+         ]
+      },
+      {
+         "name" : "Book of the New Sun",
+         "creator" : {
+            "genres" : [
+               "scifi",
+               "fantasy"
+            ],
+            "name" : "Gene Wolfe",
+            "from_country" : "USA"
+         },
+         "books" : [
+            {
+               "name" : "Shadow of the Torturer",
+               "id" : "S003.1"
+            },
+            {
+               "id" : "S003.2",
+               "name" : "Claw of the Conciliator",
+               "price" : 6.99
+            }
+         ],
+         "id" : "S003",
+         "genres" : [
+            "scifi",
+            "fantasy"
+         ]
+      },
+      {
+         "books" : [
+            {
+               "name" : "Blah",
+               "id" : "S004.1"
+            }
+         ],
+         "id" : "S004",
+         "name" : "Example with single book",
+         "creator" : {
+            "name" : "Ms Writer",
+            "genres" : [
+               "romance"
+            ],
+            "from_country" : "USA"
+         }
+      },
+      {
+         "id" : "S005",
+         "creator" : {
+            "from_country" : "USA",
+            "genres" : [
+               "romance",
+               "scifi",
+               "fantasy"
+            ],
+            "name" : "Mr Unproductive"
+         },
+         "name" : "Example with no books"
+      }
+   ]
+}

examples/books1.json

@@ -1,4 +1,5 @@
 import os
+import io
 import sys
 import logging
 import click
@@ -7,13 +8,21 @@
 from json_flattener import flatten_to_csv, unflatten_from_csv, GlobalConfig, KeyConfig, Serializer
 
 
-def _get_format(input: str, input_format: str =None) -> str:
+def _get_format(input: str, input_format: str =None, default_format: str = None) -> str:
     if input_format is None:
+        if input is None:
+            if default_format is not None:
+                return default_format
+            else:
+                raise Exception(f'Must pass input file or default format')
         _, ext = os.path.splitext(input)
         if ext is not None:
             input_format = ext.replace('.', '')
         else:
-            raise Exception(f'Must pass format option OR use known file suffix: {input}')
+            if default_format is not None:
+                return default_format
+            else:
+                raise Exception(f'Must pass format option OR use known file suffix: {input}')
     return input_format.lower()
 
 def _is_xsv(fmt: str) -> bool:
@@ -78,7 +87,7 @@ def _get_config(serializer: str = 'json', serialized_keys = [], multivalued_keys
     "-o", "--output", help="Output file, e.g. a SSSOM tsv file."
 )
 output_format_option = click.option(
-    "-O",
+    "-t",
     "--output-format",
     help=f'Desired output format, e.g. {",".join(FORMATS)}',
 )
@@ -119,6 +128,16 @@ def _get_config(serializer: str = 'json', serialized_keys = [], multivalued_keys
     multiple=True,
     help="Key configuration. Must be of form KEY={yaml,json,flat,multivalued}*",
 )
+load_config_option = click.option(
+    "-c",
+    "--load-config",
+    help="Path to global configuration file to be loaded",
+)
+save_config_option = click.option(
+    "-O",
+    "--save-config",
+    help="Path to global configuration file to be saved",
+)
 @click.group()
 @click.option("-v", "--verbose", count=True)
 @click.option("-q", "--quiet")
@@ -154,9 +173,12 @@ def main(verbose: int, quiet: bool):
 @serializer_option
 @serialized_keys_option
 @config_option
+@load_config_option
+@save_config_option
 @key_option
 def flatten(input: str, output: str, input_format: str, output_format: str, key: str,
             serializer: str, serialized_keys = [], multivalued_keys = [], flatten_keys = [],
+            save_config: str = None, load_config: str = None,
             config_key = []):
     """Flatten a file to TSV/CSV
 
@@ -197,6 +219,9 @@ def flatten(input: str, output: str, input_format: str, output_format: str, key:
     logging.debug(f'CONFIG={config}')
     with open(output, 'w') as stream:
         flatten_to_csv(objs, stream, config=config)
+    if save_config is not None:
+        with open(save_config , 'w') as stream:
+            yaml.dump(config.as_dict(), stream)
 
 
 @main.command()
@@ -209,23 +234,29 @@ def flatten(input: str, output: str, input_format: str, output_format: str, key:
 @serializer_option
 @serialized_keys_option
 @config_option
+@load_config_option
 @key_option
 def unflatten(input: str, output: str, input_format: str, output_format: str, key: str,
-            serializer: str, serialized_keys = [], multivalued_keys = [], flatten_keys = [],
-            config_key = []):
+              serializer: str, serialized_keys = [], multivalued_keys = [], flatten_keys = [],
+              load_config: str = None,
+              config_key = []):
     """Unflatten a file from TSV/CSV
 
     Example:
         jfl unflatten --input my.tsv --output my.yaml
 
     """
     input_format = _get_format(input, input_format)
-    output_format = _get_format(output, output_format)
+    output_format = _get_format(output, output_format, 'json')
     config = _get_config(serializer = serializer,
                          serialized_keys = serialized_keys,
                          multivalued_keys = multivalued_keys,
                          flatten_keys = flatten_keys,
                          config_keys = config_key)
+    if load_config is not None:
+        with open(load_config) as stream:
+            config = GlobalConfig.from_dict(**yaml.safe_load(stream))
+    logging.debug(f'CONFIG={config}')
     with open(input) as stream:
         if input_format == 'tsv':
             sep = '\t'
@@ -246,5 +277,6 @@ def unflatten(input: str, output: str, input_format: str, output_format: str, ke
         else:
             json.dump(obj, stream)
 
+
 if __name__ == "__main__":
     main()