[Draft] feat: Add sos manifest loading

insights/core/context.py

@@ -1,7 +1,9 @@
 import logging
 import os
 import six
+
 from contextlib import contextmanager
+
 from insights.util import streams, subproc
 
 log = logging.getLogger(__name__)
@@ -241,6 +243,47 @@ class SosArchiveContext(ExecutionContext):
     marker = "sos_commands"
 
 
+@fs_root
+class SerializedSosArchiveContext(ExecutionContext):
+    marker = "sos_reports/manifest.json"
+
+    @classmethod
+    def handles(cls, files):
+        if cls.marker is None or not files:
+            return None, None
+
+        sep = os.path.sep
+        m = sep + cls.marker.lstrip(sep)
+        marker_root = set()
+        for f in files:
+            if m in f:
+                i = f.find(m)
+                if f.endswith(m) or f[i + len(m)] == sep:
+                    import json
+                    root = os.path.dirname(f[:i + 1])
+                    with open(os.path.join(root, cls.marker)) as fobj:
+                        try:
+                            from packaging.version import parse as v_parse
+                        except ImportError:
+                            from distutils.version import LooseVersiof as v_parse
+
+                        manifest = json.load(fobj)
+                        if v_parse(manifest['version']) > v_parse("4.4"):
+                            marker_root.add(root)
+                        else:
+                            return None, None
+        if len(marker_root) == 1:
+            return marker_root.pop(), cls
+        if len(marker_root) > 1:
+            # when more marker found, return the one which is closest to root
+            closest_root = marker_root.pop()
+            for left_one in marker_root:
+                if len(left_one) < len(closest_root):
+                    closest_root = left_one
+            return closest_root, cls
+        return None, None
+
+
 @fs_root
 class ClusterArchiveContext(ExecutionContext):
     pass

insights/core/hydration.py

@@ -3,7 +3,7 @@
 
 from insights.core import archives, dr
 from insights.core.context import (ClusterArchiveContext, ExecutionContextMeta, HostArchiveContext,
-                                   SerializedArchiveContext)
+                                   SerializedArchiveContext, SerializedSosArchiveContext)
 from insights.core.exceptions import InvalidArchive
 from insights.core.serde import Hydration
 
@@ -71,4 +71,7 @@ def initialize_broker(path, context=None, broker=None):
     if isinstance(ctx, SerializedArchiveContext):
         h = Hydration(ctx.root)
         broker = h.hydrate(broker=broker)
+    elif isinstance(ctx, SerializedSosArchiveContext):
+        h = Hydration(ctx.root, meta_data="sos_reports/manifest.json")
+        broker = h.hydrate_sos(broker=broker)
     return ctx, broker

insights/core/serde.py

@@ -184,6 +184,79 @@ def hydrate(self, broker=None):
                 log.warning(ex)
         return broker
 
+    def _hydrate_sos_one(self, tag, file_paths):
+        """
+        This function checks for tags that start with insights_,
+        then confirms it's a valid component and generates a valid
+        results to return to be added to the broker.
+
+        Args:
+            tag (str): The sos tag used to match with the spec.
+            file_paths (list): List of files with the tag.
+
+        Returns:
+            tuple: The component, and the formatted results.
+        """
+        comp = dr.get_component_by_name("insights.specs.Specs.{0}".format(tag))
+        if comp is None:
+            return None, None
+
+        if len(file_paths) > 1:
+            if comp.multi_output:
+                results = []
+                for f in file_paths:
+                    results.append({
+                        "type": "insights.core.spec_factory.TextFileProvider",
+                        "object": {
+                            "relative_path": f,
+                            "rc": None
+                        }
+                    })
+            else:
+                results = {
+                    "type": "insights.core.spec_factory.TextFileProvider",
+                    "object": {
+                        "relative_path": file_paths[0],
+                        "rc": None
+                    }
+                }
+        elif len(file_paths) == 1:
+            results = {
+                "type": "insights.core.spec_factory.TextFileProvider",
+                "object": {
+                    "relative_path": file_paths[0],
+                    "rc": None
+                }
+            }
+        else:
+            return None, None
+
+        return comp, results
+
+    def hydrate_sos(self, broker=None):
+        """
+        Loads a Broker from a previously saved one. A Broker is created if one
+        isn't provided. Then the results from a sos manifest are loaded into the
+        broker for loading of the spec files.
+        """
+        broker = broker or dr.Broker()
+        with open(self.meta_data) as f:
+            manifest = ser.load(f)
+
+        for tag, file_paths in manifest['components']['report']['tag_summary'].items():
+            comp, results = self._hydrate_sos_one(tag, file_paths)
+            if comp is not None:
+                if not broker.get(comp):
+                    if comp.multi_output:
+                        broker[comp] = [unmarshal(results, root=self.root)]
+                    else:
+                        broker[comp] = unmarshal(results, root=self.root)
+                else:
+                    if comp.multi_output:
+                        broker[comp].append(unmarshal(results, root=self.root))
+
+        return broker
+
     def dehydrate(self, comp, broker):
         """
         Saves a component in the given broker to the file system.

insights/tests/test_serde.py

@@ -195,6 +195,20 @@ def test_hydrate_one():
     assert ser_time == 0.05
 
 
+def test_sos_hydrate_one():
+    h = Hydration()
+    comp, result = h._hydrate_sos_one("abrt_status_bare",
+                                      ["sos_commands/abrt/abrt-cli_status"])
+    assert comp == dr.get_component_by_name("insights.specs.Specs.abrt_status_bare")
+    assert result == {
+        "type": "insights.core.spec_factory.TextFileProvider",
+        "object": {
+            "relative_path": "sos_commands/abrt/abrt-cli_status",
+            "rc": None
+        }
+    }
+
+
 def test_hydrate_one_multiple_results():
     raw = {
         "name": dr.get_name(thing),