Add Python language support and fix processors intialization

.gitignore

@@ -102,4 +102,5 @@ ENV/
 
 # Ramile project ignores
 test-output*
+.idea
 .DS_Store

README.md

@@ -55,6 +55,7 @@ Ramile automatically loads the config file `.ramileconfig.json` from the project
 | CSS         | .css, .less, .sass    |
 | Swift       | .swift                |
 | Objective-C | .m                    |
+| Python      | .py                   |
 
 ## Test:
 

ramile/file_info.py

@@ -3,6 +3,7 @@ class FileInfo(object):
     file_extension = ''
     file_path = ''
     is_in_comment_block = False
+    comment_block_end_sign = None
     blank_lines = 0
     comment_lines = 0
     extracted_lines = 0
@@ -28,10 +29,12 @@ def found_comment_line(self):
         self.comment_lines += 1
         return
 
-    def mark_comment_block_start(self):
+    def mark_comment_block_start(self, block_end_sign):
         self.is_in_comment_block = True
+        self.comment_block_end_sign = block_end_sign
         return
 
     def mark_comment_block_end(self):
         self.is_in_comment_block = False
+        self.comment_block_end_sign = None
         return

ramile/filters/comment_block_filter.py

@@ -0,0 +1,63 @@
+from ramile.processors import LineFilterBase
+
+
+class CommentBlockFilterBase(LineFilterBase):
+    """ Filtering out comment block """
+    block_signs = {}
+
+    def filter(self, file, line):
+        if file.is_in_comment_block:
+            file.found_comment_line()
+            if self.close_comment_block(file, line):
+                file.mark_comment_block_end()
+            return line, True
+        else:
+            is_comment_block, comment_block_start_sign, comment_block_end_sign = self.is_comment_block(line)
+            if is_comment_block:
+                file.mark_comment_block_start(comment_block_end_sign)
+                file.found_comment_line()
+                line_tail = line[len(comment_block_start_sign):]
+                if self.close_comment_block(file, line_tail):
+                    file.mark_comment_block_end()
+                return line, True
+        return line, False
+
+    def is_comment_block(self, line):
+        """ determine whether current line starts a comment block
+
+        :param line: current line
+        :return is_comment_block: whether current line starts a comment block
+        :return comment_block_end_sign: sign to close the comment block
+        """
+        for sign_start in self.block_signs.keys():
+            if line.startswith(sign_start):
+                return True, sign_start, self.block_signs[sign_start]
+        return False, None, None
+
+    def close_comment_block(self, file, line):
+        return line.endswith(file.comment_block_end_sign)
+
+
+class PythonCommentBlockFilter(CommentBlockFilterBase):
+    """ Filtering out Python multi-line docstrings with \"\"\" and '''
+    """
+    block_signs = {
+        '"""': '"""',
+        "'''": "'''"
+    }
+
+
+class CStyleCommentBlockFilter(CommentBlockFilterBase):
+    """ Filters out C-style comment blocks with '/*' and '*/'
+    """
+    block_signs = {
+        '/*': '*/'
+    }
+
+
+class HtmlCommentBlockFilter(CommentBlockFilterBase):
+    """ Filters out html comment blocks with '<!--' and '-->'
+    """
+    block_signs = {
+        '<!--': '-->'
+    }

ramile/filters/comment_line_filter.py

@@ -0,0 +1,38 @@
+from ramile.processors import LineFilterBase
+
+
+class CommentFilterBase(LineFilterBase):
+    """ Filters out single line comments which start with '//'
+    """
+    comment_sign = None
+    strip_endings = False
+
+    def filter(self, file, line):
+        if line.startswith(self.comment_sign):
+            file.found_comment_line()
+            return line, True
+        if not line:
+            return line, False
+        if self.strip_endings:
+            if self.comment_sign in line:
+                return line[:line.index(self.comment_sign)], False
+        return line, False
+
+
+class DoubleSlashCommentFilter(CommentFilterBase):
+    """ Filters out single line comments which start with '//'
+    """
+    comment_sign = '//'
+    strip_endings = True
+
+
+class SharpCommentFilter(CommentFilterBase):
+    """ Filters out single line comments which start with '#'
+    """
+    comment_sign = '#'
+
+
+class ColonCommentFilter(CommentFilterBase):
+    """ Filters out single line comments which start with ':'
+    """
+    comment_sign = ':'

ramile/processors/__init__.py

@@ -5,10 +5,10 @@ class FileProcessorBase(object):
     """ Base class for file processors. The processor for each lanuage should inherit from this class.
     """
     expected_extensions = []
-    filters = []
 
     def __init__(self):
         # by default, processors of all languages will always starts with a blank line filter
+        self.filters = []
         self.filters.append(BlankLineFilter())
         return
 
@@ -49,7 +49,7 @@ class LineFilterBase(object):
     """ A filter will process each line, and determine whether each line should be dropped. A filter can also perform any neccessary process on the line and replace the original line.
     """
 
-    def filter(self, line):
+    def filter(self, file, line):
         """ Filters a line of code, outputs the filtered content, and a flag whether the line should be dropped.
         """
         return line, False
@@ -83,6 +83,10 @@ def __build_processors(self):
         self.__cache_processor(CssProcessor())
         self.__cache_processor(SwiftProcessor())
         self.__cache_processor(OCProcessor())
+        self.__cache_processor(PythonProcessor())
+        self.__cache_processor(CppProcessor())
+        self.__cache_processor(JsonProcessor())
+        self.__cache_processor(BatchProcessor())
         self.__cache_processor(GoProcessor())
         return
 
@@ -96,13 +100,16 @@ def __cache_processor(self, processor):
         return
 
 
-from ramile.processors.blank_line_filter import BlankLineFilter
-from ramile.processors.comment_block_filter import CommentBlockFilter
+from ramile.filters.blank_line_filter import BlankLineFilter
 from ramile.processors.js_processor import JsProcessor
 from ramile.processors.java_processor import JavaProcessor
 from ramile.processors.php_processor import PhpProcessor
 from ramile.processors.html_processor import HtmlProcessor
 from ramile.processors.css_processor import CssProcessor
 from ramile.processors.swift_processor import SwiftProcessor
 from ramile.processors.oc_processor import OCProcessor
+from ramile.processors.python_processor import PythonProcessor
+from ramile.processors.cpp_processor import CppProcessor
+from ramile.processors.json_processor import JsonProcessor
+from ramile.processors.bat_processor import BatchProcessor
 from ramile.processors.go_processor import GoProcessor

ramile/processors/bat_processor.py

@@ -0,0 +1,20 @@
+#! /usr/bin/env python
+#
+# Copyright 2021 eVision.ai Inc. All Rights Reserved.
+#
+# @author: Chen Shijiang([email protected])
+# @date: 2021-01-04 19:53
+# @version: 1.0
+#
+from ramile.filters.comment_line_filter import ColonCommentFilter, SharpCommentFilter
+from ramile.processors import FileProcessorBase
+
+
+class BatchProcessor(FileProcessorBase):
+    expected_extensions = ['.bat', '.cmd']
+
+    def __init__(self):
+        super().__init__()
+        self.filters.append(SharpCommentFilter())
+        self.filters.append(ColonCommentFilter())
+        return

ramile/processors/cpp_processor.py

@@ -0,0 +1,12 @@
+from ramile.filters.comment_block_filter import CStyleCommentBlockFilter
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
+from ramile.processors import FileProcessorBase
+
+
+class CppProcessor(FileProcessorBase):
+    expected_extensions = ['.cpp', '.c', 'h']
+
+    def __init__(self):
+        super().__init__()
+        self.filters.append(DoubleSlashCommentFilter())
+        self.filters.append(CStyleCommentBlockFilter())

ramile/processors/css_processor.py

@@ -1,15 +1,13 @@
+from ramile.filters.comment_block_filter import CStyleCommentBlockFilter
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
 from ramile.processors import FileProcessorBase
-from ramile.processors import BlankLineFilter
-from ramile.processors.c_style_comment_block_filter import CStyleCommentBlockFilter
-from ramile.processors.double_slash_comment_filter import DoubleSlashCommentFilter
-from ramile.processors.html_comment_block_filter import HtmlCommentBlockFilter
 
 
 class CssProcessor(FileProcessorBase):
     expected_extensions = ['.css', '.less', '.sass']
 
     def __init__(self):
-        self.filters.append(BlankLineFilter())
+        super().__init__()
         self.filters.append(CStyleCommentBlockFilter())
         self.filters.append(DoubleSlashCommentFilter())
         return

ramile/processors/html_processor.py

@@ -1,15 +1,13 @@
+from ramile.filters.comment_block_filter import CStyleCommentBlockFilter, HtmlCommentBlockFilter
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
 from ramile.processors import FileProcessorBase
-from ramile.processors import BlankLineFilter
-from ramile.processors.c_style_comment_block_filter import CStyleCommentBlockFilter
-from ramile.processors.double_slash_comment_filter import DoubleSlashCommentFilter
-from ramile.processors.html_comment_block_filter import HtmlCommentBlockFilter
 
 
 class HtmlProcessor(FileProcessorBase):
     expected_extensions = ['.html', '.htm']
 
     def __init__(self):
-        self.filters.append(BlankLineFilter())
+        super().__init__()
         self.filters.append(CStyleCommentBlockFilter())
         self.filters.append(DoubleSlashCommentFilter())
         self.filters.append(HtmlCommentBlockFilter())

ramile/processors/java_processor.py

@@ -1,14 +1,13 @@
+from ramile.filters.comment_block_filter import CStyleCommentBlockFilter
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
 from ramile.processors import FileProcessorBase
-from ramile.processors import BlankLineFilter
-from ramile.processors.c_style_comment_block_filter import CStyleCommentBlockFilter
-from ramile.processors.double_slash_comment_filter import DoubleSlashCommentFilter
 
 
 class JavaProcessor(FileProcessorBase):
     expected_extensions = ['.java']
 
     def __init__(self):
-        self.filters.append(BlankLineFilter())
+        super().__init__()
         self.filters.append(CStyleCommentBlockFilter())
         self.filters.append(DoubleSlashCommentFilter())
         return

ramile/processors/js_processor.py

@@ -1,14 +1,13 @@
+from ramile.filters.comment_block_filter import CStyleCommentBlockFilter
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
 from ramile.processors import FileProcessorBase
-from ramile.processors import BlankLineFilter
-from ramile.processors.c_style_comment_block_filter import CStyleCommentBlockFilter
-from ramile.processors.double_slash_comment_filter import DoubleSlashCommentFilter
 
 
 class JsProcessor(FileProcessorBase):
     expected_extensions = ['.js', '.jsx', '.vue', '.wpy']
 
     def __init__(self):
-        self.filters.append(BlankLineFilter())
+        super().__init__()
         self.filters.append(CStyleCommentBlockFilter())
         self.filters.append(DoubleSlashCommentFilter())
         return

ramile/processors/json_processor.py

@@ -0,0 +1,19 @@
+#! /usr/bin/env python
+#
+# Copyright 2021 eVision.ai Inc. All Rights Reserved.
+#
+# @author: Chen Shijiang([email protected])
+# @date: 2021-01-04 19:53
+# @version: 1.0
+#
+from ramile.filters.comment_line_filter import DoubleSlashCommentFilter
+from ramile.processors import FileProcessorBase
+
+
+class JsonProcessor(FileProcessorBase):
+    expected_extensions = ['.json', '.gradle']
+
+    def __init__(self):
+        super().__init__()
+        self.filters.append(DoubleSlashCommentFilter())
+        return