Multivarian

PantheonCMD/pcbuild.py

@@ -8,6 +8,7 @@
 import subprocess
 import threading
 import yaml
+from pcyamlchecks import variant_yaml_validation
 
 lock = threading.Lock()
 
@@ -21,16 +22,9 @@ def build_content(content_files, lang, repo_location, yaml_file_location):
 
     current_count = 0
 
-    # Parse the main YAML file
-    with open(yaml_file_location, 'r') as f:
-        main_yaml_file = yaml.safe_load(f)
+    attributes = variant_yaml_validation(yaml_file_location)
+
 
-    for item, members in main_yaml_file.items():
-        if item == 'variants':
-            attributes_file_location = repo_location + members[0]["path"]
-            with open(attributes_file_location,'r') as attributes_file:
-                attributes = parse_attributes(attributes_file.readlines())
-            break
     try:
         pool = concurrent.futures.ThreadPoolExecutor()
         futures = []
@@ -52,20 +46,6 @@ def build_content(content_files, lang, repo_location, yaml_file_location):
     return True
 
 
-def parse_attributes(attributes):
-    """Read an attributes file and parse values into a key:value dictionary."""
-
-    final_attributes = {}
-
-    for line in attributes:
-        if re.match(r'^:\S+:.*', line):
-            attribute_name = line.split(":")[1].strip()
-            attribute_value = line.split(":")[2].strip()
-            final_attributes[attribute_name] = attribute_value
-
-    return final_attributes
-
-
 def copy_resources(resources):
     """Copy resources such as images and files to the build directory."""
     script_dir = os.path.dirname(os.path.realpath(__file__))
@@ -117,7 +97,7 @@ def coalesce_document(main_file, attributes=None, depth=0, top_level=True):
     attributes = attributes or {}
     comment_block = False
     lines = []
-    
+
     # Create a copy of global attributes
     if top_level:
         attributes_global = attributes.copy()

PantheonCMD/pcyamlchecks.py

@@ -2,12 +2,14 @@
 
 import os
 import sys
+import glob
+import re
 import yaml
+import pprint
 from cerberus import Validator, errors
 from cerberus.errors import BasicErrorHandler
 from pcchecks import Regex, icons_check, toc_check
 from pcvalidator import Report
-import glob
 
 
 class CustomErrorHandler(BasicErrorHandler):
@@ -18,10 +20,32 @@ class CustomErrorHandler(BasicErrorHandler):
     messages[errors.NOT_NULLABLE.code] = "value can't be empty"
 
 
+class Printing():
+
+    def __init__(self):
+        self.report = {}
+        self.count = 0
+
+    def create_report(self, message, items):
+        """Generate report."""
+        self.count += 1
+        if not message in self.report:
+            self.report[message] = []
+        self.report[message].append(items)
+
+    def print_report(self):
+        separator = "\n\t"
+
+        for message, items in self.report.items():
+            print('\nFAIL: {} contains the following files or directories that do not exist in your repository:\n'.format(message))
+            for item in items:
+                print('\t' + separator.join(item))
+
+
 def get_yaml_size(yaml_file):
     """Test if pv2.yml is empty."""
     if os.path.getsize(yaml_file) == 0:
-        print("\nYour pantheon2.yml file is empty; exiting...")
+        print("\nYour {} file is empty; exiting...".format(yaml_file))
         sys.exit(2)
 
 
@@ -31,56 +55,86 @@ def load_doc(yaml_file):
         try:
             return yaml.safe_load(file)
         except yaml.YAMLError:
-            print("There's a syntax error in your pantheon2.yml file. Please fix it and try again.\nTo detect an error try running yaml lint on your pantheo2.yml file.")
+            print("There's a syntax error in your {0} file. Please fix it and try again.\nTo detect an error try running yaml lint on your {0} file.".format(yaml_file))
             sys.exit(2)
 
 
-def get_attribute_file_validation_results(attribute_file):
-    """Validate attributes file."""
-    report = Report()
-
-    for path in attribute_file:
-        with open(path, 'r') as file:
-            original = file.read()
-            stripped = Regex.MULTI_LINE_COMMENT.sub('', original)
-            stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped)
-
-            icons_check(report, stripped, path)
-            toc_check(report, stripped, path)
-
-    return report
-
-
-def get_yaml_errors(yaml_schema, yaml_doc):
+def get_yaml_errors(yaml_schema, yaml_doc, path):
     # load validator with custom error handler
     v = Validator(yaml_schema, error_handler=CustomErrorHandler())
     # validate the pv2.yml with schema
     v.validate(yaml_doc, yaml_schema)
 
     if v.errors:
-        print("\nFAIL: there is an error in your yaml file:")
+        print("\nFAIL: there is an error in your {} file:".format(path))
         for key in v.errors.keys():
             print("\n\t'{}' {}".format(key, ', '.join(str(item) for item in v.errors[key])))
+            sys.exit(2)
+
+
+def get_resources_existence(yaml_doc):
+    path_does_not_exist = []
+
+    for item in yaml_doc['resources']:
+        path_to_images_dir = os.path.split(item)[0]
+        if not glob.glob(path_to_images_dir):
+            path_does_not_exist.append(item)
+
+    if path_does_not_exist:
+        print('\nFAIL: Your pantheon2.yml contains the following files or directories that do not exist in your repository:\n')
+        for path in path_does_not_exist:
+            print('\t', path)
         sys.exit(2)
 
-    else:
 
-        path_does_not_exist = []
-        path_exists = []
+def get_variants_yaml(yaml_doc):
+    variants_do_not_exist = []
+    variants_exist = []
 
-        for item in yaml_doc['resources']:
-            path_to_images_dir = os.path.split(item)[0]
-            if not glob.glob(path_to_images_dir):
-                path_does_not_exist.append(item)
+    for item in yaml_doc['variants']:
+        if not glob.glob(item):
+            variants_do_not_exist.append(item)
+            print('\nFAIL: {} contains the following files or directories that do not exist in your repository:\n'.format('variants yaml'))
+            for path in variants_do_not_exist:
+                print('\t', path)
+            sys.exit(2)
+
+        else:
+            for path in glob.glob(item):
+                variants_exist.append(path)
+
+    return variants_exist
+
+
+def get_attribute_files(var_yaml_doc):
+    path_does_not_exist = []
+    path_exists = []
+
+    for item in var_yaml_doc['path']:
+        if not os.path.exists(item):
+            path_does_not_exist.append(item)
+            print('\nFAIL: {} contains the following files or directories that do not exist in your repository:\n'.format('variants yaml'))
+            for path in path_does_not_exist:
+                print('\t', path)
+            sys.exit(2)
+        else:
+            path_exists.append(item)
 
-        for variant in yaml_doc['variants']:
-            if not os.path.exists(variant['path']):
-                path_does_not_exist.append(variant['path'])
-            else:
-                path_exists.append(variant['path'])
+    return path_exists
+
+
+def get_variant_attribute_path(yaml_var_doc):
+    path_does_not_exist = []
+    path_exists = []
+
+    for item in yaml_var_doc['path']:
+        if not os.path.exists(item):
+            path_does_not_exist.append(item)
+        else:
+            path_exists.append(item)
 
     if path_does_not_exist:
-        print('\nFAIL: Your pantheon2.yml contains the following files or directories that do not exist in your repository:\n')
+        print('\nFAIL: Your {} contains the following files or directories that do not exist in your repository:\n'.format('variants yaml'))
         for path in path_does_not_exist:
             print('\t', path)
         sys.exit(2)
@@ -89,6 +143,40 @@ def get_yaml_errors(yaml_schema, yaml_doc):
         if attribute_file_validation.count != 0:
             print("\nYour attributes file has the following errors:\n")
             attribute_file_validation.print_report()
+        else:
+            return path_exists
+
+
+def get_attribute_file_validation(attribute_file):
+    """Validate attributes file."""
+    report = Report()
+    attributes = []
+
+    for path in attribute_file:
+        with open(path, 'r') as file:
+            original = file.read()
+            stripped = Regex.MULTI_LINE_COMMENT.sub('', original)
+            stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped)
+
+            icons_check(report, stripped, path)
+            toc_check(report, stripped, path)
+
+        with open(path, 'r') as file:
+            attributes.append(parse_attributes(file.readlines()))
+
+    return report, attributes
+
+
+def get_attribute_file_validation_results(attribute_file):
+    report, attributes = get_attribute_file_validation(attribute_file)
+
+    return report
+
+
+def get_attributes(attribute_file):
+    report, attributes = get_attribute_file_validation(attribute_file)
+
+    return attributes
 
 
 def yaml_validation(yaml_file):
@@ -101,4 +189,97 @@ def yaml_validation(yaml_file):
     loaded_yaml = load_doc(yaml_file)
 
     get_yaml_size(yaml_file)
-    get_yaml_errors(schema, loaded_yaml)
+    get_yaml_errors(schema, loaded_yaml, yaml_file)
+    get_resources_existence(loaded_yaml)
+
+
+def parse_attributes(attributes):
+    """Read an attributes file and parse values into a key:value dictionary."""
+
+    final_attributes = {}
+
+    for line in attributes:
+        if re.match(r'^:\S+:.*', line):
+            attribute_name = line.split(":")[1].strip()
+            attribute_value = line.split(":")[2].strip()
+            final_attributes[attribute_name] = attribute_value
+
+    return final_attributes
+
+
+def variant_yaml_validation(yaml_file):
+    # define path to script
+    path_to_script = os.path.dirname(os.path.realpath(__file__))
+    loaded_yaml = load_doc(yaml_file)
+    var_yaml = get_variants_yaml(loaded_yaml)
+    schema2 = eval(open(path_to_script + '/schemavar.py', 'r').read())
+
+    variant_yaml_files = get_variants_yaml(loaded_yaml)
+
+    result = {}
+
+    for item in variant_yaml_files:
+        get_yaml_size(item)
+
+        with open(item, 'r') as file:
+            try:
+                var_yaml = yaml.safe_load(file)
+
+                get_yaml_errors(schema2, var_yaml, item)
+                attributes_exist = get_variant_attribute_path(var_yaml)
+
+                if attributes_exist:
+                    result[item] = get_attributes(attributes_exist)
+
+            except yaml.YAMLError:
+                print("There's a syntax error in your {0} file. Please fix it and try again.\nTo detect an error try running yaml lint on your {0} file.".format(item))
+                sys.exit(2)
+
+    return result
+
+
+pattern = re.compile('\{([^}]+)\}')
+
+original = variant_yaml_validation('/home/levi/rhel-8-docs/pantheon2.yml')
+
+
+pattern = re.compile('\{([^}]+)\}')
+
+
+def resolve_attributes(text, dictionary):
+    while True:
+        attributes = pattern.findall(text)
+
+        if not any(key in dictionary for key in attributes):
+            return text
+
+        for attribute in attributes:
+            if attribute in dictionary:
+                text = text.replace(f'{{{attribute}}}', dictionary[attribute])
+
+
+def resolve_dictionary(dictionary):
+    result = {}
+
+    for key, value in dictionary.items():
+        result[key] = resolve_attributes(value, dictionary)
+
+    return result
+
+
+def resolve_all(dictionary):
+    result = {}
+
+    for yml_file, items in dictionary.items():
+        new_items = []
+
+        for i in items:
+            new_items.append(resolve_dictionary(i))
+
+        result[yml_file] = new_items
+
+    return result
+
+
+'''pp = pprint.PrettyPrinter(width=41, compact=True)
+pp.pprint(resolve_all(original))'''

PantheonCMD/schema.py

@@ -10,23 +10,6 @@
     'variants': {
         'required': True,
         'type': 'list',
-        'schema': {
-            'type': 'dict',
-            'schema': {
-                'name': {
-                    'required': True,
-                    'type': 'string'
-                },
-                'path': {
-                    'required': True,
-                    'type': 'string'
-                },
-                'canonical': {
-                    'required': True,
-                    'allowed': [True, False]
-                }
-            }
-        }
     },
     'assemblies': {
         'required': True,