Merge pull request #133 from kevinrizza/add-filename-to-logs

Add filename to warnings and errors

Author: kevinrizza

operatorcourier/build.py

@@ -1,3 +1,4 @@
+import os
 import yaml
 import operatorcourier.identify as identify
 
@@ -9,26 +10,58 @@ def _get_empty_bundle(self):
                 customResourceDefinitions=[],
                 clusterServiceVersions=[],
                 packages=[],
+            ),
+            metadata=dict(
+                filenames=dict()
             )
         )
 
     def _get_field_entry(self, yamlContent):
         yaml_type = identify.get_operator_artifact_type(yamlContent)
         return yaml_type[0:1].lower() + yaml_type[1:] + 's'
 
-    def _updateBundle(self, operatorBundle, yamlContent):
-        field_entry = operatorBundle["data"][self._get_field_entry(yamlContent)]
-        field_entry.append(yaml.safe_load(yamlContent))
+    def _get_relative_path(self, path):
+        """
+        :param path: the path of the file
+        :return: the file name along with its parent folder
+        """
+        path = os.path.normpath(path)
+        parts = path.split(os.sep)
+        return os.path.join(parts[-2], parts[-1])
+
+    def _updateBundle(self, operatorBundle, file_name, yaml_string):
+        # Determine which operator file type the yaml is
+        operator_artifact = self._get_field_entry(yaml_string)
+
+        # Marshal the yaml into a dictionary
+        yaml_data = yaml.safe_load(yaml_string)
+
+        # Add the data dictionary to the correct list
+        operatorBundle["data"][operator_artifact].append(yaml_data)
+
+        # Encode the dictionary into a string, then use that as a key to reference
+        # the file name associated with that yaml file. Then add it to the metadata.
+        if file_name != "":
+            unencoded_yaml = yaml.dump(yaml_data)
+            relative_path = self._get_relative_path(file_name)
+            operatorBundle["metadata"]["filenames"][hash(unencoded_yaml)] = relative_path
+
         return operatorBundle
 
-    def build_bundle(self, strings):
-        """build_bundle takes an array of yaml files and generates a 'bundle'
+    def build_bundle(self, bundle_data):
+        """build_bundle takes bundle_data, a list of yaml files and
+        associated metadata, and generates a 'bundle'
         with those yaml files generated in the bundle format.
 
-        :param strings: Array of yaml strings to bundle
+        :param bundle_data: Array of tuples consisting of yaml blobs
+        and associated metadata for those blobs
         """
+        # Generate an empty bundle
         bundle = self._get_empty_bundle()
-        for item in strings:
-            bundle = self._updateBundle(bundle, item)
+
+        # For each file, append the file to the right place in the bundle
+        # and add the associated metadata to the metadata field
+        for data in bundle_data:
+            bundle = self._updateBundle(bundle, data[0], data[1])
 
         return bundle

operatorcourier/validate.py

@@ -1,6 +1,7 @@
 import logging
 import json
 import semver
+import yaml
 
 import validators as v
 
@@ -10,11 +11,19 @@
     metadata_annotations_required_fields,
     spec_required_fields)
 
+log_info = {'current_manifest_file': ''}
 logger = logging.getLogger(__name__)
+logger.propagate = False
+handler = logging.StreamHandler()
+formatter = logging.Formatter('%(levelname)s: %(message)s [%(current_manifest_file)s]')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger = logging.LoggerAdapter(logger, log_info)
 
 
 class ValidateCmd():
     dataKey = "data"
+    metadataKey = "metadata"
     crdKey = "customResourceDefinitions"
     csvKey = "clusterServiceVersions"
     pkgsKey = "packages"
@@ -46,6 +55,17 @@ def _log_error(self, message, *args, **kwargs):
         self.validation_json['errors'].append(message % args)
         logger.error(message, *args, **kwargs)
 
+    def get_filename_from_metadata(self, metadata, yaml_dict):
+        """
+        :param yaml_dict: The yaml dictionary associated with a filename
+        :return: Filename associated with a yaml dictionary
+        """
+        filename = ""
+        hashed_key = hash(yaml.dump(yaml_dict))
+        if hashed_key in metadata["filenames"]:
+            filename = metadata["filenames"][hashed_key]
+        return filename
+
     def validate(self, bundle, repository=None):
         """validate takes a bundle as a dictionary and returns a boolean value that
         describes if the bundle is valid. It also logs verification information when
@@ -69,15 +89,15 @@ def _bundle_validation(self, bundle, repository=None):
 
         bundleData = bundle[self.dataKey]
 
-        validationDict[self.crdKey] = self._type_validation(bundleData, self.crdKey,
+        validationDict[self.crdKey] = self._type_validation(bundle, self.crdKey,
                                                             self._crd_validation, False)
-        validationDict[self.csvKey] = self._type_validation(bundleData, self.csvKey,
+        validationDict[self.csvKey] = self._type_validation(bundle, self.csvKey,
                                                             self._csv_validation, True)
-        validationDict[self.pkgsKey] = self._type_validation(bundleData, self.pkgsKey,
+        validationDict[self.pkgsKey] = self._type_validation(bundle, self.pkgsKey,
                                                              self._pkgs_validation, True)
         if self.ui_validate_io:
             validationDict[self.csvKey] &= self._type_validation(
-                bundleData, self.csvKey, self._ui_validation_io, True)
+                bundle, self.csvKey, self._ui_validation_io, True)
         if validationDict[self.pkgsKey] and repository is not None:
             packageName = bundleData['packages'][0]['packageName']
             if repository != packageName:
@@ -93,13 +113,18 @@ def _bundle_validation(self, bundle, repository=None):
 
         return valid
 
-    def _crd_validation(self, bundleData):
+    def _crd_validation(self, bundle):
         logger.info("Validating custom resource definitions.")
         valid = True
 
+        bundle_metadata = bundle[self.metadataKey]
+        bundleData = bundle[self.dataKey]
         crds = bundleData[self.crdKey]
 
         for crd in crds:
+            crd_file_name = self.get_filename_from_metadata(bundle_metadata, crd)
+            log_info['current_manifest_file'] = crd_file_name
+
             if "metadata" in crd:
                 if "name" in crd["metadata"]:
                     logger.info("Evaluating crd %s", crd["metadata"]["name"])
@@ -137,13 +162,18 @@ def _crd_validation(self, bundleData):
 
         return valid
 
-    def _csv_validation(self, bundleData):
+    def _csv_validation(self, bundle):
         valid = True
         logger.info("Validating cluster service versions.")
 
+        bundle_metadata = bundle[self.metadataKey]
+        bundleData = bundle[self.dataKey]
         csvs = bundleData[self.csvKey]
 
         for csv in csvs:
+            csv_file_name = self.get_filename_from_metadata(bundle_metadata, csv)
+            log_info['current_manifest_file'] = csv_file_name
+
             if "metadata" in csv:
                 if self._csv_metadata_validation(csv["metadata"]) is False:
                     valid = False
@@ -357,10 +387,12 @@ def _csv_metadata_validation(self, metadata):
 
         return valid
 
-    def _pkgs_validation(self, bundleData):
+    def _pkgs_validation(self, bundle):
         valid = True
         logger.info("Validating packages.")
 
+        bundle_metadata = bundle[self.metadataKey]
+        bundleData = bundle[self.dataKey]
         pkgs = bundleData[self.pkgsKey]
 
         num_pkgs = len(pkgs)
@@ -370,6 +402,8 @@ def _pkgs_validation(self, bundleData):
             return False
 
         pkg = pkgs[0]
+        pkg_file_name = self.get_filename_from_metadata(bundle_metadata, pkg)
+        log_info['current_manifest_file'] = pkg_file_name
 
         if "packageName" in pkg:
             logger.info("Evaluating package %s", pkg["packageName"])
@@ -409,11 +443,13 @@ def _pkgs_validation(self, bundleData):
 
         return valid
 
-    def _type_validation(self, bundleData, typeName, validator, required):
+    def _type_validation(self, bundle, typeName, validator, required):
+        bundleData = bundle[self.dataKey]
+
         valid = True
         if typeName in bundleData:
             if len(bundleData[typeName]) != 0:
-                valid = validator(bundleData)
+                valid = validator(bundle)
             elif required:
                 self._log_error("Bundle does not contain any %s." % typeName)
                 valid = False
@@ -423,10 +459,11 @@ def _type_validation(self, bundleData, typeName, validator, required):
 
         return valid
 
-    def _ui_validation_io(self, bundleData):
+    def _ui_validation_io(self, bundle):
         valid = True
         logger.info("Validating cluster service versions for operatorhub.io UI.")
 
+        bundleData = bundle[self.dataKey]
         csvs = bundleData[self.csvKey]
 
         for csv in csvs:

operatorcourier/verified_manifest.py

@@ -25,7 +25,8 @@ def __init__(self, source_dir, yamls, ui_validate_io, repository):
         self.nested = False
 
         if yamls:
-            manifests = {FLAT_KEY: yamls}
+            yaml_strings_with_metadata = self._set_empty_filepaths(yamls)
+            manifests = {FLAT_KEY: yaml_strings_with_metadata}
         else:
             manifests = self.get_manifests_info(source_dir)
 
@@ -34,6 +35,14 @@ def __init__(self, source_dir, yamls, ui_validate_io, repository):
             self.get_validation_dict_from_manifests(manifests, ui_validate_io, repository)
         self.is_valid = False if self.__validation_dict['errors'] else True
 
+    def _set_empty_filepaths(self, yamls):
+        yaml_strings_with_metadata = []
+        for yaml_string in yamls:
+            yaml_tuple = ("", yaml_string)
+            yaml_strings_with_metadata.append(yaml_tuple)
+
+        return yaml_strings_with_metadata
+
     def get_manifests_info(self, source_dir):
         """
         Given a source directory, this method returns a dict containing all
@@ -68,22 +77,18 @@ def get_manifests_info(self, source_dir):
             for manifest_path in manifest_paths:
                 manifest_dir_name = os.path.basename(manifest_path)
 
-                files_content = []
                 crd_files_info, csv_files_info = get_crd_csv_files_info(manifest_path)
-                for (_, file_content) in (crd_files_info + csv_files_info):
-                    files_content.append(file_content)
-                manifests[manifest_dir_name] = files_content
+                manifests[manifest_dir_name] = crd_files_info + csv_files_info
             for manifest_dir_name in manifests:
-                manifests[manifest_dir_name].append(pkg_path_and_content[1])
+                manifests[manifest_dir_name].append(pkg_path_and_content)
         # flat layout: collect all valid manifest files and add to FLAT_KEY entry
         elif pkg_path_and_content and csvs_path_and_content:
             logger.info('The source directory is in flat structure.')
             crd_files_info, csv_files_info = get_crd_csv_files_info(root_path)
-            files_content = [pkg_path_and_content[1]]
-            files_content.extend(
-                [file_content for (_, file_content) in (crd_files_info + csv_files_info)])
+            files_info = [pkg_path_and_content]
+            files_info.extend(crd_files_info + csv_files_info)
 
-            manifests[FLAT_KEY] = files_content
+            manifests[FLAT_KEY] = files_info
         else:
             msg = 'The source directory structure is not in valid flat or nested format,'\
                   'because no valid CSV file is found in root or manifest directories.'
@@ -108,8 +113,8 @@ def get_validation_dict_from_manifests(self, manifests, ui_validate_io=False,
         bundle_dict = None
         # validate on all bundles files and combine log messages
         validation_dict = ValidateCmd(ui_validate_io).validation_json
-        for version, manifest_files_content in manifests.items():
-            bundle_dict = BuildCmd().build_bundle(manifest_files_content)
+        for version, manifest_files_info in manifests.items():
+            bundle_dict = BuildCmd().build_bundle(manifest_files_info)
             if version != FLAT_KEY:
                 logger.info("Parsing version: %s", version)
             _, validation_dict_temp = ValidateCmd(ui_validate_io, self.nested) \

tests/test_build.py

@@ -10,7 +10,10 @@ def test_create_bundle():
     yamls = []
     for path in paths:
         with open(path) as f:
-            yamls.append(f.read())
+            yaml_data = f.read()
+            filename = ""
+            yaml = (filename, yaml_data)
+            yamls.append(yaml)
 
     bundle = BuildCmd().build_bundle(yamls)
     assert bool(bundle["data"]["packages"]) is True