yasi.py

#!/usr/bin/env python
# coding: utf-8

""" yasi

Date: 20th November 2013
Author: nkmathew <kipkoechmathew@gmail.com>

Dialect aware s-expression indenter

"""

from __future__ import print_function

import argparse
import hashlib
import os
import re
import shutil
import sys
import time
import collections
import json
import difflib

try:
    from functools import lru_cache
except ImportError:
    from backports.functools_lru_cache import lru_cache

# pylint: disable=unused-import
from pprint import pprint  # noqa

__version__ = '2.1.2'


@lru_cache(maxsize=None)
def create_args_parser():
    """ Return command line parser """
    parser = argparse.ArgumentParser(
        description='Dialect-aware s-expression indenter', prog='yasi')
    parser.add_argument('files', help='List of files to be indented. '
                        'Will indent from standard input if no files are specified',
                        nargs='*')
    parser.add_argument(
        '-nc', '--no-compact', '--nc', dest='compact',
        help='Do not compact the code, just indent', action='store_false')
    parser.add_argument(
        '-nb', '--no-backup', '--nb', dest='backup', action='store_false',
        help='Do not create a backup file even if --backup-dir is specified ')
    parser.add_argument(
        '-nm', '--no-modify', '--nm', dest='modify',
        help='Do not modify the file', action='store_false')
    parser.add_argument(
        '--diff', '-diff', dest='output_diff',
        help='Prints unified diff of the initial and final result',
        action='store_true')
    parser.add_argument(
        '-nw', '--no-warning', '--nw', dest='warning',
        help='Do not display warnings', action='store_false')
    parser.add_argument(
        '-nr', '--no-rc', '--nr', dest='read_rc',
        help='Ignore any rc files in the current or home folder',
        action='store_false')
    parser.add_argument(
        '--no-output', '-no-output', dest='output',
        help='Suppress output of the indented code', action='store_false')
    parser.add_argument(
        '-c', '--color', '-color', dest='colour_diff',
        help='Display diff text in color', action='store_true')
    parser.add_argument(
        '-ne', '--no-exit', '--ne', dest='exit', action='store_false',
        help='Instructs the program not to exit when a warning is raised.')
    parser.add_argument(
        '-o', dest='output_file',
        help='Path/name of output file', type=str, default='')
    parser.add_argument(
        '--tab', '-tab', dest='tab_size',
        help='Indent with tabs using the specified tabwidth. A tab is assumed \
        equal to 4 spaces by default when expanding the tabs in the input file',
        type=int, default=-1)
    parser.add_argument(
        '--dialect', '-dialect',
        help='Use Scheme keywords', type=str, default='')
    parser.add_argument(
        '-v', '--version', action='version',
        help='Prints script version', version='yasi v%s' % __version__)
    parser.add_argument(
        '-suffix', '--suffix', dest='backup_suffix', help='Backup file suffix',
        type=str, default='.yasi.bak~')
    parser.add_argument(
        '-bd', '--backup-dir', '--bd', '-backup-dir',
        help='The directory where the backup file is to be written',
        type=str, default=os.getcwd())
    parser.add_argument(
        '-is', '--indent-size', '--is',
        help='The number of spaces per indent',
        type=int, default=2)
    parser.add_argument(
        '-di', '--default-indent', '--di',
        help='The indent level to be used in case a '
        "function's argument is in the next line. Vim uses 2, the most common being 1.",
        type=int, default=1)
    parser.add_argument(
        '-ic', '--indent-comments', '--ic',
        help='If true, comment lines will be indented possibly '
        'messing with any deliberate comment layout', action='store_true')
    parser.add_argument(
        '-uni', '--uniform', '-uniform', '--uni',
        help='Dictates whether the if-clause and else-clause of an if-like'
        'block should have the same indent level.',
        action='store_true')
    parser.add_argument(
        '-parallel', '--parallel',
        help='Process the given files in parallel',
        action='store_true')
    return parser


def parse_args(arguments=None):
    """ Reads command-line arguments

    >>> parse_args('--indent-comments')
    """
    if arguments is None:
        arguments = sys.argv[1:]
    if isinstance(arguments, str):
        arguments = arguments.split()
    if isinstance(arguments, argparse.Namespace):
        return arguments
    parser = create_args_parser()
    args = parser.parse_args(arguments)

    args.dialect = args.dialect.lower()
    if args.dialect not in ['lisp', 'newlisp', 'clojure', 'scheme', 'all', '']:
        parser.error("`{0}' is not a recognized dialect".format(args.dialect))

    args.backup_dir = os.path.expanduser(args.backup_dir)
    if not os.path.exists(args.backup_dir):
        parser.error("Directory `{0}' does not exist".format(args.backup_dir))

    if len(args.files) > 1 and args.output_file:
        parser.error('Cannot use the -o flag when more than one file is specified')

    if not args.files:
        # Indentation from standard input
        if args.modify and not args.output_file:
            args.modify = False
        args.backup = False
        args.warning = False

    if args.output_diff:
        # If someone requests a diff we assume they don't want the file to be
        # modified
        args.modify = False

    return args


def read_file(fname):
    """ read_file(fname : str) -> str

    >>> read_file(r'C:\\mine\\test.lisp')
    r'(print "No, no, there\'s \\r\\nlife in him!. ")\\r\\n\\r\\n'

    The file is read in binary mode in order to preserve original line endings.
        Line ending    Binary mode Text mode
            CRLF            CRLF    LF
            CR              CR      LF
    """
    assert os.path.exists(fname), "\n--%s-- Warning: File `%s' does not exist..." \
        % (current_time(), fname)
    with open(fname, 'rb') as fp:
        return fp.read().decode('utf-8')


def current_time():
    """ current_time() -> str

    >>> current_time()
    14:28:04

    Returns the current local time in 24 clock system.
    """
    return time.strftime('%X', (time.localtime()))


def backup_source_file(fname, args=None):
    """ backup_source_file(fname : str)

    >>> backup_source_file('~/Desktop/lisp/test.lisp')

    Create a backup copy of the source file.
    """
    args = parse_args(args)
    backup_dir = args.backup_dir
    assert os.path.exists(fname), \
        ("\n--%s-- Warning: File `%s' does not exist..." % (current_time(), fname))
    assert os.path.exists(os.path.abspath(backup_dir)), \
        ("\n--%s-- Warning: Directory `%s' does not exist..." % (current_time(), fname))
    backup_name = backup_dir + os.sep + os.path.split(fname)[1] + args.backup_suffix
    try:
        shutil.copyfile(fname, backup_name)
    except IOError:
        message = "\n--%s-- Warning: Couldn't backup the file `%s' in `%s', check if you have enough permissions. "
        tpl = (current_time(), fname, backup_dir)
        sys.stderr.write(message % tpl)


def md5sum(content):
    """ md5sum(content : str) -> str

    >>> md5sum('Keyboard not found!! Press F1 to continue...')
    'ad98cde09016d2e99a726966a4291acf'

    Returns a checksum to be used to determine whether the file has changed.
    A simple textual comparison can still do the work
    """
    return hashlib.md5(content).hexdigest()


def find_line_ending(string):
    """ find_line_ending(string : str) -> str

    >>> find_line_ending('Elementary my dear Watson. \\r')
    '\\r'

    Find the line ending in the file so that we can try to preserve it.
    """
    if CRLF in string:
        return CRLF
    if CR in string:
        return CR
    return LF


@lru_cache(maxsize=None)
def trim(string):
    """ trim(string : str) -> str

    Uses every usefull hack to try and reduce extra whitespace without
    messing with character literals
    """
    # Trailing whitespace
    string = re.sub('[ \t]*$', '', string)
    # turn '(print(+ 1 1))' to '(print (+ 1 1))'
    string = re.sub(r'''([^\\(\[, {@~`'#^])(\(|\[|{)''', r'\1 \2', string, re.X)
    # turn  ')(' to ') ('
    string = re.sub(r'(\)|\]|})(\[|\(|{)', r'\1 \2', string)
    # Remove any space before closing brackets '(print 12   )' ==> '(print 12)'
    string = re.sub('[ \t]*(\)|\]|})', r'\1', string)
    # remove extra whitespace "(print     'this)" ==> "(print 'this)"
    string = re.sub('[ \t]{2,}', ' ', string)
    # turn ') ) ) ' into '))) '
    string = re.sub(r'(\))[ \t]*(?=(\)))', r'\1', string)
    string = re.sub(r'(\])[ \t]*(?=(\]))', r'\1', string)
    string = re.sub(r'(})[ \t]*(?=(}))', r'\1', string)
    # turn '( ( ( ' into '((( '
    string = re.sub(r'(\()[ \t]*(?=(\())', r'\1', string)
    string = re.sub(r'(\[)[ \t]*(?=(\[))', r'\1', string)
    string = re.sub(r'({)[ \t]*(?=({))', r'\1', string)
    # remove leading whitespace '   print' ==> 'print'
    string = re.sub('^[ \t]*', '', string)
    # Remove space between quote and opening bracket, "' (1 2 3)" ==> "'(1 2 3)"
    string = re.sub("('|`)[ \t]+(\(|\[|{)", r'\1\2', string)
    return string


def find_trim_limit(string, args=None):
    """ find_trim_limit(string : str) -> int

    >>> find_trim_limit(r'(list #\; #\")')
    14
    >>> find_trim_limit(r'(list ; ")')
    6
    >>> find_trim_limit(r'(list " ;)')
    7

    The function attempts to identify upto which point we are supposed to trim
    so that we don't mess with strings or any aligned comments.

    It does this by comparing the positions of semicolons and double
    quotes. It doesn't consider the multiline comment marker. If your
    code uses multiline comments(#| ... |#), you'll have to use --no-compact mode
    """
    args = parse_args(args)
    # Find position of the first unescaped semi colon
    comment_start = re.search(r'([^\\];)|(^;)', string)
    # Find position of the first unescaped double quote
    string_start = re.search(r'([^\\]")|(^")', string)
    # Assign -1 if there's no match
    limit = string_start.end() if string_start else -1
    comment_start = comment_start.end() if comment_start else -1
    if comment_start != -1:
        # If a semi colon is found, include all the whitespace before it to preserve
        # any aligned comments
        comment_start = re.search('[ \t]*;', string).start()

    if args.dialect == 'newlisp':
        # Find out which string type comes first(normal, tag or brace strings)
        brace_string_start = re.search('{', string)
        tag_string_start = re.search('\[text\]', string)
        brace_string_start = brace_string_start.end() if brace_string_start else -1
        tag_string_start = tag_string_start.end() if tag_string_start else -1
        pos_lst = [limit, brace_string_start, tag_string_start]
        pos_lst = [x for x in pos_lst if x != -1]
        if pos_lst:
            limit = min(pos_lst)

    if comment_start != -1 and limit != -1:
        if comment_start < limit:
            # If the semicolon comes before the comma, it means the string has been
            # commented out
            limit = comment_start
    elif comment_start != -1 and limit == -1:
        # If there's a semicolon but no quote, use the semicolon position as the
        # limit
        limit = comment_start
    elif limit == -1:
        # If neither a semicolon nor a double quote has been found, use the length
        # of the string as the limit
        limit = len(string)
    return limit


@lru_cache(maxsize=None)
def is_macro_name(func_name, dialect):
    """ is_macro_name(func_name : str, dialect : str) -> bool

    >>> is_macro_name('yacc:define-parser')
    True

    Tests if a word is a macro using the language's/dialect's convention,
    e.g macros in Lisp usually start with 'def' and 'with' in Scheme. Saves
    the effort of finding all the macros in Lisp/Scheme/Clojure/newLISP and storing
    them in a list.
    """
    if not func_name:
        return False
    if dialect == 'lisp':
        return re.search('^(macro|def|do|with-)', func_name, re.I)
    if dialect == 'scheme':
        return re.search('^(call-|def|with-)', func_name)
    if dialect == 'clojure':
        return re.search('^(def|with)', func_name)
    if dialect == 'newlisp':
        return re.search('^(macro|def)', func_name)
    return False


@lru_cache(maxsize=None)
def split_preserve(string, sep):
    """ split_preserve(string : str, sep : str)  -> [str]

    >>> split_preserve('''
    "My dear Holmes, " said I, "this is too much. You would certainly
    have been burned, had you lived a few centuries ago.
                ''', '\\n')
    ['\\n',
     '    "My dear Holmes, " said I, "this is too much. You would certainly\\n',
     '    have been burned, had you lived a few centuries ago.\\n',
     '                ']

    Splits the string and sticks the separator back to every string in the list.
    """
    # split the whole string into a list so that you can iterate line by line.
    str_list = string.split(sep)
    if str_list[-1] == '':
        # If you split 'this\nthat\n' you get ['this', 'that', ''] if
        # you add newlines to every string in the list you get
        # ['this\n', 'that\n', '\n']. You've just added
        # another newline at the end of the file.
        del str_list[-1]
        str_list = [x + sep for x in str_list]
    else:
        # ['this', 'that'] will become ['this\n', 'that\n'] when
        # mapped. A newline has been added to the file. We don't want
        # this, so we strip it below.
        str_list     = [x + sep for x in str_list]
        str_list[-1] = str_list[-1].rstrip(sep)
    return str_list


@lru_cache(maxsize=None)
def all_whitespace(string):
    """ all_whitespace(string : str) -> bool

    >>> all_whitespace('      ')
    True

    Returns True if a string has only whitespace.
    """
    return re.search('^[ \t]*(\r|\n|$)', string)


def detabify(text, args):
    """ tabify(text : str, args : argparse.Namespace|str) -> str

    Expands tabs
    """
    args = parse_args(args)
    if args.tab_size < 1:
        return text.expandtabs(4)
    return text.expandtabs(args.tab_size)


def tabify(text, args):
    """ tabify(text : str, args : argparse.Namespace|str) -> str

    >>> tabify('        (println "hello world")', '--tab=3')
     '\t\t  (println "hello world")'

    Replace spaces with tabs
    """
    args = parse_args(args)
    if args.tab_size < 1:
        return text
    tab_equiv = ' ' * args.tab_size
    return text.replace(tab_equiv, '\t')


def pad_leading_whitespace(string, zero_level, blist, args=None):
    """ pad_leading_whitespace(string : str, current_level : int,
                               zero_level : int) -> str

    >>> pad_leading_whitespace("(print 'Yello)")
    "         (print 'Yello)"

    Takes a string and indents it using the current indentation level
    and the zero level.
    """
    args = parse_args(args)
    if args.compact:
        # if compact mode is on, split the string into two, trim the first
        # position and merge the two portions.
        trim_limit = find_trim_limit(string, args)
        comment_line = re.search('^[ \t]*;', string, re.M)
        if comment_line and args.indent_comments:
            trim_limit = comment_line.end()
        substr1 = string[0:trim_limit]
        substr2 = string[trim_limit:]
        substr1 = trim(substr1)
        string = substr1 + substr2
    else:
        # If in nocompact mode, remove leading spaces only
        string = re.sub('^[ \t]+', '', string, count=0)

    indent_level = zero_level
    if blist:
        indent_level = blist[-1]['indent_level']

    padding = ' ' * indent_level
    padding = tabify(padding, args)
    return padding + string, indent_level


def indent_line(zerolevel, bracket_list, line, in_comment, in_symbol_region,
                args=None):
    """ indent_line(zerolevel : int, bracket_list : list, line : str, in_comment : bool,
                    in_symbol_region : bool, args : string|list)

    Most important function in the indentation process. It uses the bracket
    locations stored in the list to indent the line.
    """
    args = parse_args(args)
    comment_line = re.search('^[ \t]*;', line, re.M)
    if args.indent_comments:
        # We are allowed to indent comment lines
        comment_line = False
    if not args.compact and bracket_list == [] and not in_comment:
        # If nocompact mode is on and there are no unclosed blocks, try to
        # find the zero level by simply counting spaces before a line that
        # is not empty or has a comment
        _line = detabify(line, args)
        leading_spaces = re.search('^[ \t]+[^; )\n\r]', _line)
        if leading_spaces:
            # NOTE: If you don't subtract one here, the zero level will increase
            # every time you indent the file because the character at the end of
            # the regex is part of the capture.
            zerolevel = leading_spaces.end() - 1
        else:
            zerolevel = 0

    if in_symbol_region:
        # No processing done in strings and comments
        return zerolevel, line, 0
    if not comment_line and not all_whitespace(line):
        # If this is not a comment line indent the line.
        # If the list is empty, then the current_level defaults
        # to zero
        curr_line, current_level = pad_leading_whitespace(line, zerolevel,
                                                          bracket_list, args)
        return zerolevel, curr_line, current_level
    return zerolevel, line, 0

# ---------------------------------------------------------------------------------
# GLOBAL CONSTANTS::


CR   = '\r'
LF   = '\n'
CRLF = CR + LF

KEYWORD0 = 0  # Non-keyword
KEYWORD1 = 1  # Indents uniformly by 1 unit
KEYWORD2 = 2  # Distinguishes subforms
KEYWORD3 = 3  # Indents uniformly by 2 units
KEYWORD4 = 4  # A 1-keyword used mostly for defining local functions e.g flets

# Keywords that indent by two spaces
SCHEME_KEYWORDS = \
    ['define', 'local-odd?', 'when', 'begin', 'case',
     'local-even?', 'do', 'call-with-bytevector-output-port',
     'call-with-input-file', 'call-with-port',
     'call-with-current-continuation', 'open-file-input-port',
     'call-with-port', 'call-with-values', 'call-with-output-file',
     'call-with-string-output-port', 'define-syntax', 'if', 'let', 'let*',
     'library', 'unless', 'lambda', 'syntax-rules', 'syntax-case',
     'let-syntax', 'letrec*', 'letrec', 'let-values', 'let*-values',
     'with-exception-handler', 'with-input-from-file',
     'with-interrupts-disabled', 'with-input-from-string',
     'with-output-to-file', 'with-input-from-port',
     'with-output-to-string', 'with-source-path', 'with-syntax',
     'with-implicit',
     'with-error-handler', 'module', 'parameterize']

CLOJURE_KEYWORDS = \
    ['defn', 'fn', 'dorun', 'doseq', 'loop', 'when',
     'let', 'defmacro', 'binding', 'doto', 'ns', ':import', 'defstruct',
     'condp', 'comment', 'when', 'when-let', '->', '->>',
     'extend-type', 'reify', 'binding', 'when-not', 'proxy', 'dotimes',
     'try', 'finally', 'for', 'letfn', 'catch', 'iterate', 'while',
     'with-local-vars', 'locking', 'defmulti', 'defmethod', 'extend'
     ]

LISP_KEYWORDS = \
    [':implementation', ':method', 'case', 'defclass',
     'defconstant', 'defgeneric', 'defimplementation',
     'define-condition', 'define-implementation-package',
     'definterface', 'defmacro', 'defmethod', 'defpackage',
     'defproject', 'deftype', 'defun', 'defvar', 'do-external-symbols',
     'dolist', 'dotimes', 'ecase', 'etypecase', 'flet', 'handler-bind',
     'if', 'lambda', 'let', 'let*', 'print-unreadable-object',
     'macrolet', 'defparameter', 'with-slots', 'typecase', 'loop', 'when', 'prog1',
     'unless', 'with-open-file', 'with-output-to-string', 'with-input-from-string',
     'block', 'handler-case', 'defstruct', 'eval-when', 'tagbody', 'ignore-errors',
     'labels', 'multiple-value-bind', 'progn', 'unwind-protect', 'collect'
     ]

NEWLISP_KEYWORDS = \
    ['while', 'if', 'case', 'dotimes', 'define', 'dolist', 'catch',
     'throw', 'lambda', 'lambda-macro', 'when', 'unless', 'letex', 'begin',
     'dostring', 'let', 'letn', 'doargs', 'define-macro', 'until', 'do-until',
     'do-while', 'for-all', 'find-all', 'for'
     ]

# The 'if' and 'else' part of an if block should have different indent levels so
# that they can stand out since there's no else Keyword in Lisp/Scheme to make
# this explicit.  list IF_LIKE helps us track these keywords.
IF_LIKE = ['if']


@lru_cache(maxsize=None)
def parse_rc_json():
    """ Reads the json configuration file(.yasirc.json), parses it and returns the
    dictionary
    """
    fname = '.yasirc.json'
    path = os.path.expanduser('~/' + fname)
    if os.path.exists(fname):
        path = os.path.abspath(fname)
    elif not os.path.exists(path):
        path = ''
    content = ''
    if path:
        with open(path) as f:
            content = f.read()
    ret = {}
    if content:
        ret = json.loads(content)
    return collections.defaultdict(dict, ret)


def assign_indent_numbers(lst, inum, dic):
    """ Associate keywords with their respective indentation numbers
    """
    for i in lst:
        dic[i] = inum
    return dic


def add_keywords(args):
    """ add_keywords(dialect : str) -> [str, str]

    Takes a lisp dialect name and returns a list of keywords that increase
    indentation by two spaces and those that can be one-armed like 'if'
    """
    dialect = args.dialect
    keywords = collections.defaultdict(int)
    two_spacers = []
    two_armed = IF_LIKE
    local_binders = []
    if dialect == 'lisp':  # Lisp
        two_spacers = LISP_KEYWORDS
        two_armed += ['multiple-value-bind', 'destructuring-bind', 'do', 'do*']
        local_binders += ['flet', 'macrolet', 'labels']
    elif dialect == 'scheme':  # Scheme
        two_spacers = SCHEME_KEYWORDS
        two_armed += ['with-slots', 'do', 'do*']
        local_binders += []
    elif dialect == 'clojure':  # Clojure
        two_spacers = CLOJURE_KEYWORDS
        two_armed += []
        local_binders += ['letfn']
    elif dialect == 'newlisp':  # newLISP
        two_spacers = NEWLISP_KEYWORDS
        two_armed += []
        local_binders += []
    elif dialect == 'all':
        two_spacers = LISP_KEYWORDS + SCHEME_KEYWORDS + CLOJURE_KEYWORDS + \
            NEWLISP_KEYWORDS
    keywords = assign_indent_numbers(two_spacers, KEYWORD1, keywords)
    keywords = assign_indent_numbers(two_armed, KEYWORD2, keywords)
    keywords = assign_indent_numbers(local_binders, KEYWORD4, keywords)
    if args.read_rc:
        rc_keywords = parse_rc_json()
        keywords.update(rc_keywords[dialect])
    return keywords

# ---------------------------------------------------------------------------------


def find_first_arg_pos(bracket_offset, curr_line, args=None):
    """ find_first_arg_pos(bracket_offset : int, curr_line : str) -> [int, int]

    Arguments:
    bracket_offset - The position of the bracket in the current line e.g
        "    (  list 'timey 'wimey  )" --> 4
        " (  list 'timey 'wimey  )"    --> 1
        "(  list 'timey 'wimey  )"     --> 0

    >>> find_first_arg_pos(0, "(     list 'one-sheep 'two-sheep )")
    [11, 5]

    Returns the position of the first argument to the function relative to the
    position of the opening bracket and the number of spaces between the opening
    bracket and the function name.
    The two values will to be used to align the other arguments in the subsequent line
    """
    args = parse_args(args)
    spaces_before_func = 0
    subline = curr_line[bracket_offset + 1:]
    if re.search('^[ \t]*($|\r)', subline):
        # whitespace extending to the end of the line means there's no
        # function in this line. The indentation level defaults to one.
        arg_pos = 1
    else:
        if bracket_offset != len(curr_line) - 1 and curr_line[bracket_offset + 1] == ' ':
            # control reaches here if we are not at the end of the line
            # and whitespace follows. We must first find the position of the
            # function and then the arguments position
            match = re.search(' +[^)\]]| \)', subline)  # Find the first non whitespace/bracket character
            if match:
                spaces_before_func = match.end() - match.start() - 1
                end = match.end()
            else:
                end = 0
            # Then use the end of the whitespace group as the first argument
            arg_pos = re.search(' +([^)])|( *(\(|\[))', subline[end:])
            if arg_pos:
                arg_pos = arg_pos.end() + spaces_before_func + 1
            else:
                arg_pos = spaces_before_func + 1
            if re.match('^[ \t]*(#\||;|$|\r)',
                        subline[(end - 1 + subline[end - 1:].find(' ')):]):
                # But, if a comment if found after the function name, the
                # indent level becomes one
                arg_pos = spaces_before_func + args.default_indent
        else:
            # If there's no space after the bracket, simply find the end of the
            # whitespace group
            match = re.search(' +([^)}\n\r])|( *(\(|\[|{))', subline)
            if match:  # found the argument
                arg_pos = match.end()
            else:  # Either empty list or argument is in the next line
                arg_pos = 1
            if re.match('^[\t ]*(;|$|\r)', subline[subline.find(' '):]):
                # Again if a comment is found after the function name, the
                # indent level defaults to 1
                arg_pos = spaces_before_func + args.default_indent
    return [arg_pos, spaces_before_func]


def _pop_from_list(bracket, lst, line, real_pos, offset, msg_stack):
    """ _pop_from_list(char : str, lst : [str], line : str,
                        real_pos : int, offset : int)

    The function is called when a closing bracket is encountered. The function
    simply pops the last pushed item and issues a warning if an error is
    encountered.
    """
    # Try to spot a case when a square bracket is used to close a round bracket
    # block
    if bracket == ']':
        correct_closer = '['
    elif bracket == ')':
        correct_closer = '('
    else:
        correct_closer = '{'
    if lst != []:
        popped = lst.pop()
        popped_char = popped['character']
        popped_pos = popped['line_number']
        popped_offset = popped['bracket_pos']
        if popped_char is not correct_closer:
            message = "Bracket `%s' does not match `%s' at (%d, %d)"
            message = message % (bracket, popped_char, popped_pos, popped_offset)
            warning_info = {
                'msg': message,
                'line': line,
                'column': real_pos
            }
            msg_stack.append(warning_info)
    else:
        # If the list is empty and a closing bracket is found, it means we have
        # excess brackets. That warning is issued here. The coordinates used
        # will be slightly or largely off target depending on how much your
        # code was modified when used with compact mode
        message = "Unmatched closing bracket `%s'" % bracket
        warning_info = {
            'msg': message,
            'line': line,
            'column': offset + 1
        }
        msg_stack.append(warning_info)
    return lst


def _push_to_list(lst, func_name, char, line, offset,
                  first_arg_pos, first_item, in_list_literal,
                  lead_spaces, args=None):
    """ _push_to_list(lst : [str], func_name : str, char : str, line : int, offset : int,
                        first_arg_pos :int , first_item : int, in_list_literal : bool,
                        lead_spaces : int, args : str)

    Called when an opening bracket is encountered. A hash containing the
    necessary data to pin point errors and the indentation level is stored in
    the list and the list returned.
    """
    args = parse_args(args)
    keywords = add_keywords(args)
    pos_hash = {'character': char,
                'line_number': line,
                'bracket_pos': offset,
                'indent_level': offset + first_arg_pos,  # the default value, e.g in normal function
                'func_name': func_name,
                'spaces': 0}

    is_macro = is_macro_name(func_name, args.dialect)
    two_spacer = is_macro or keywords[func_name] in [KEYWORD1, KEYWORD4]

    if in_list_literal or char == '{' or (char == '[' and args.dialect == 'clojure'):
        # found quoted list or clojure hashmap/vector
        pos_hash['indent_level'] = first_item

    elif keywords[func_name] == KEYWORD2:
        # We only make the if-clause stand out if not in uniform mode
        pos_hash['indent_level'] = lead_spaces + ((offset + args.indent_size * 2)
                                                  if not args.uniform
                                                  else (offset + args.indent_size))

    elif func_name != '':
        if two_spacer:
            pos_hash['indent_level'] = lead_spaces + offset + args.indent_size
        elif keywords[func_name] == KEYWORD3:
            pos_hash['indent_level'] = lead_spaces + offset + (2 * args.indent_size)

    lst.append(pos_hash)
    try:
        # A hack to make flets and labels in Lisp not indent like
        # functions. The 'labels' indentation may not be exactly
        # perfect.
        parent_func = lst[-3]['func_name']
        # Make 'special' indentation occur only in a Clojure binding block([]) for
        # letfns
        non_bind_block = args.dialect == 'clojure' and lst[-2]['character'] != '['
        if keywords[parent_func] == KEYWORD4 and not non_bind_block:
            lst[-1]['indent_level'] = offset + args.indent_size
    except IndexError:
        pass
    return lst


def indent_code(original_code, args=None):
    """ indented_code(string : str, fname : str) -> [...]

    Arguments:
    fpath: Simply used in formatting the warning messages

    >>> indent_code("(print\n'Hello)")
    {'bracket_locations': [],
     'comment_locations': [],
     'in_comment': 0,
     'in_newlisp_tag_string': False,
     'in_string': False,
     'in_symbol_with_space': False,
     'indented_code': ['(print\n', " 'Hello)"],
     'last_quote_location': (),
     'last_symbol_location': (),
     'message_stack': [],
     'newlisp_brace_locations': [],
     'original_code': ['(print\n', "'Hello)"],
     'first_tag_string': ()}


    The last entry in the list is the indented string.
    """

    args = parse_args(args)
    keywords = add_keywords(args)

    # Safeguards against processing brackets inside strings
    in_string = False

    # newLISP use curly brackets as a syntax for multiline strings
    # this variable here tries to keep track of that
    in_newlisp_string = 0
    in_newlisp_tag_string = False
    newlisp_brace_locations = []
    first_tag_string = ()

    # zero_level helps us get the same results as Sitaram's indenter when in
    # --no-compact mode.
    zero_level = 0

    # The two variables prevent formatting comment regions or symbols with whitespace
    in_comment = 0
    in_symbol_with_space = False
    comment_locations = []
    last_symbol_location = ()

    # A in_symbol_region is the region between pipes(|   |) or in strings. This
    # includes the comment region. This region is not to be messed with.
    in_symbol_region = in_string or in_comment or in_symbol_with_space or \
        in_newlisp_string or in_newlisp_tag_string

    # we need to know the line number in order to issue almost accurate messages about
    # unclosed brackets and string
    line_number = 1

    # Stores the last position a quote was encountered so that in case there are
    # any unclosed strings, we can pinpoint them
    last_quote_location = ()

    line_ending = find_line_ending(original_code)
    code_lines = split_preserve(original_code, line_ending)

    indented_code = []

    bracket_locations = []

    # List of warnings from errors in the code
    message_stack = []

    for line in code_lines:
        escaped   = False
        curr_line = line

        # Get the indent level and the indented line
        zero_level, curr_line, indent_level = indent_line(zero_level,
                                                          bracket_locations,
                                                          line, in_comment,
                                                          in_symbol_region, args)
        # Build up the indented string.
        indented_code.append(curr_line)
        regex = '^[ \t]*'
        lead_spaces = re.findall(regex, curr_line)
        if lead_spaces:
            curr_line = re.sub(regex, detabify(lead_spaces[0], args), curr_line)
        offset = 0
        for curr_char in curr_line:
            next_char = curr_line[offset + 1:offset + 2]
            prev_char = curr_line[offset - 1:offset]

            substr = curr_line[offset + 1:]  # slice to the end

            if escaped:
                # Move to the next character if the current one has been escaped
                escaped = False
                offset += 1
                continue

            if curr_char == '\\' and not in_newlisp_string and not in_newlisp_tag_string:
                # the next character has been escaped
                escaped = True

            if (curr_char == ';' or (curr_char == '#' and args.dialect == 'newlisp'))\
                    and not in_symbol_region and not \
                    (prev_char == '#' and args.dialect == 'scheme'):
                # a comment has been found, go to the next line
                # A sharp sign(#) before a semi-colon in Scheme is used to
                # comment out sections of code. We don't treat it as a comment
                break

            # ----------------------------------------------------------
            # Comments are dealt with here. Clojure and newLISP don't have Lisp
            # style multiline comments so don't include them.
            if args.dialect not in ['clojure', 'newlisp'] and curr_char == '|' \
                    and not in_string:
                if prev_char == '#' and not in_symbol_with_space:
                    comment_locations.append((line_number, offset))
                    in_comment += 1
                elif in_comment and next_char == '#':
                    in_comment -= 1
                    comment_locations.pop()
                elif not in_comment:
                    if in_symbol_with_space:
                        last_symbol_location = ()
                        in_symbol_with_space = False
                    else:
                        last_symbol_location = (line_number, offset)
                        in_symbol_with_space = True

            # ----------------------------------------------------------

            # Strings are dealt with here only if we are not in a comment
            if not (in_symbol_with_space or in_comment or in_newlisp_tag_string):
                if curr_char == '"':
                    last_quote_location = (line_number, offset)
                    in_string = not bool(in_string)
                if args.dialect == 'newlisp' and not in_string:
                    # We handle newLISP's multiline(brace) string here. Brace
                    # strings can nest
                    if curr_char == '{':
                        newlisp_brace_locations.append((line_number, offset))
                        in_newlisp_string += 1
                    elif curr_char == '}':
                        if newlisp_brace_locations:
                            newlisp_brace_locations.pop()
                        else:
                            message = "Attempt to close a non-existent newLISP string"
                            warning_info = {
                                'msg': message,
                                'line': line_number,
                                'column': offset
                            }
                            message_stack.append(warning_info)
                        in_newlisp_string -= 1

            if curr_char == '[' and args.dialect == 'newlisp' and not \
                    (in_newlisp_string or in_string):
                # We have to handle tag strings in newLISP here.
                if re.match('\[text\]', curr_line[offset:offset + 7]):
                    in_newlisp_tag_string = True
                    if first_tag_string == ():
                        first_tag_string = (line_number, offset)
                elif re.match('\[/text\]', curr_line[offset:offset + 7]):
                    in_newlisp_tag_string = False
                    first_tag_string = ()

            in_symbol_region = in_string or in_comment or in_symbol_with_space \
                or in_newlisp_string or in_newlisp_tag_string

            if in_symbol_region:
                # move on if we are in a string, a symbol with a space or a comment
                # altogether known as the symbol region
                offset += 1
                continue

            # Finds the real position of a bracket to be used in pinpointing where
            # the unclosed bracket is. The real position is different from the offset
            # because current offset is the position of the bracket in the
            # trimmed string not the original.
            real_position = (offset - zero_level) + \
                len(re.findall('^[ \t]*', line)[0]) - indent_level
            if curr_char in ['(', '[', '{']:
                if curr_char in ['[', '{'] and args.dialect in ['lisp', 'newlisp']:
                    # Square/Curly brackets are used should not contribute to
                    # the indentation in CL and newLISP
                    offset += 1
                    continue

                first_arg_pos, spaces_before_func = \
                    find_first_arg_pos(offset, curr_line, args)
                func_name = substr[0:first_arg_pos - 1].strip(')]\t\n\r ').lower()
                in_list_literal = False
                if re.search("[^#]('|`|#)([ \t]*\(|\[)($|\r)", curr_line[0:offset + 1]):
                    in_list_literal = True

                if re.search('^[^ \t]+[ \t]*($|\r)', substr):
                    # The function is the last symbol/form in the line
                    func_name = substr.strip(')]\t\n\r ').lower()

                if in_list_literal:
                    # an empty string is always in a non-empty string, we don't want
                    # this. We set False as the func_name because it's not a string
                    # in_list_literal prevents an keyword in a list literal from
                    # affecting the indentation
                    func_name = ''

                if func_name in ['define-macro', 'defmacro']:
                    # Macro names are part of two space indenters.
                    # This part tries to find the name so that it is not indented
                    # like a function the next time it's used.
                    end_of_space = re.search('^[ \t]*', substr).end()
                    substr = substr[end_of_space:]
                    substr = substr[re.search('[ \t]*', substr).start():].strip()
                    macro_name = substr[:substr.find(' ')]  # macro name is delimeted by whitespace
                    if macro_name != '':
                        keywords[macro_name] = KEYWORD1

                # first_item stores the position of the first item in the literal list
                # it's necessary so that we don't assume that the first item is always
                # after the opening bracket.
                first_item = re.search('[ \t]*', curr_line[offset + 1:]).end() + offset + 1
                bracket_locations = _push_to_list(bracket_locations[:], func_name,
                                                  curr_char, line_number, offset,
                                                  first_arg_pos, first_item,
                                                  in_list_literal,
                                                  spaces_before_func, args)

            elif curr_char in [']', ')', '}']:
                if curr_char in [']', '}'] and args.dialect in ['lisp', 'newlisp']:
                    # Square/Curly brackets are used should not contribute to
                    # the indentation in CL and newLISP
                    offset += 1
                    continue

                bracket_locations = _pop_from_list(curr_char, bracket_locations[:],
                                                   line_number, real_position,
                                                   offset, message_stack)

            if bracket_locations and curr_char in [' ', '\t'] and \
                    keywords[bracket_locations[-1]['func_name']] == KEYWORD2:
                # This part changes the indentation level of a then clause so that
                # we can achieve something like:
                #         (if (= this that)
                #             'then-form
                #           'else-form)
                # This is done by keeping track of the number of spaces found. If
                # you find two spaces it means that, for example that we have just
                # passed the then-form and hence should decrease the indentation
                # level by 2.(I shamelessly copied this algorithm from Dorai's
                # indenter)
                if prev_char not in [' ', '\t', ''] or not \
                        re.search('^[ \t]*(;|#\||$|\r)', curr_line):
                    # The level shouldn't be decreased if the line is a comment
                    # line. The regex above takes care of that.
                    bracket_locations[-1]['spaces'] += 1
                if bracket_locations[-1]['spaces'] == 2:
                    bracket_locations[-1]['indent_level'] -= \
                        0 if args.uniform else args.indent_size
                    # some dummy value to prevent control from reaching here again
                    bracket_locations[-1]['spaces'] = 999

            offset += 1
        line_number += 1
    res = {
        'message_stack': message_stack,
        'first_tag_string': first_tag_string,
        'in_newlisp_tag_string': in_newlisp_tag_string,
        'last_symbol_location': last_symbol_location,
        'comment_locations': comment_locations,
        'newlisp_brace_locations': newlisp_brace_locations,
        'in_string': in_string,
        'in_comment': in_comment,
        'in_symbol_with_space': in_symbol_with_space,
        'bracket_locations': bracket_locations,
        'last_quote_location': last_quote_location,
        'original_code': code_lines,
        'indented_code': indented_code
    }
    return res


def colour_diff(diff_lines):
    """ colour_diff(diff_lines : lst)

    Print diff text to terminal in color
    """

    try:
        import colorama
    except ImportError:
        # colorama is not available, print plain diff
        print(''.join(list(diff_lines)))
        return

    colorama.init()

    def p_green(text):
        """ Print added line in green """
        print(colorama.Fore.GREEN + text + colorama.Fore.WHITE, end='')

    def p_yellow(text):
        """ Print diff section header in yellow """
        print(colorama.Fore.YELLOW + text + colorama.Fore.WHITE, end='')

    def p_red(text):
        """ Print removed line in red """
        print(colorama.Fore.RED + text + colorama.Fore.WHITE, end='')
    section = re.compile('@@\s+-\d\d,\d\d\s\+\d\d,\d\d\s+@@')
    for line in diff_lines:
        if line.startswith('-'):
            p_red(line)
        elif line.startswith('+'):
            p_green(line)
        elif section.search(line):
            p_yellow(line)
        else:
            print(line, end='')


def _post_indentation(res, args=None, fpath=''):
    """ _post_indentation(res : dict):

    Called after the string has been indented appropriately.
    It takes care of writing the file and checking for unclosed strings
    or comments.
    """
    fname = os.path.basename(fpath)
    args = parse_args(args)

    for msg in res['message_stack']:
        if args.warning:
            if args.files:
                msg['fname'] = fname
                sys.stderr.write('\n{fname}:{line}:{column}: {msg}'.format(**msg))
            else:
                # Input was passed through stdin
                sys.stderr.write('\n:{line}:{column}: {msg}'.format(**msg))

    if res['bracket_locations']:
        # If the bracket_locations list is not empty it means that there are some
        # brackets(opening) that haven't been closed.
        for bracket in res['bracket_locations']:
            line = bracket['line_number']
            column = bracket['bracket_pos']
            character = bracket['character']
            # The bracket_locations are not very accurate. The warning might be
            # misleading because it considers round and square brackets to be
            # the same.
            message = "\n%s:%d:%d: Unmatched `%s'"
            if args.warning:
                sys.stderr.write(message % (fname, line, column, character))

    if res['newlisp_brace_locations']:
        for brace in res['newlisp_brace_locations']:
            message = "\n%s:%d:%d: Unclosed newLISP brace string"
            if args.warning:
                sys.stderr.write(message % (fname, brace[0], brace[1]))

    if res['comment_locations']:
        for comment in res['comment_locations']:
            message = "\n%s:%d:%d: Unclosed multiline comment"
            tpl = (fname,) + comment
            if args.warning:
                sys.stderr.write(message % tpl)

    if res['last_symbol_location']:
        message = "\n%s:%d:%d: Unclosed symbol"
        tpl = (fname,) + res['last_symbol_location']
        if args.warning:
            sys.stderr.write(message % tpl)

    if res['in_string']:
        message = "\n%s:%d:%d: String extends to end-of-file"
        tpl = (fname,) + res['last_quote_location']
        if args.warning:
            sys.stderr.write(message % tpl)

    if res['in_newlisp_tag_string']:
        message = "\n%s:%d:%d: Tag string extends to end-of-file"
        tpl = (fname,) + res['first_tag_string']
        if args.warning:
            sys.stderr.write(message % tpl)

    output_file = args.output_file
    if not output_file:
        output_file = fpath

    indented_code = res['indented_code']
    indent_result = ''.join(indented_code)
    if indented_code == res['original_code'] and args.files:
        message = "File '%s' has already been formatted. Leaving it unchanged...\n"
        sys.stderr.write(message % fname)
        if output_file != fpath:
            with open(output_file, 'wb') as indented_file:
                indented_file.write(indent_result.encode('utf8'))
    else:
        if args.output_diff:
            diff = difflib.unified_diff(res['original_code'], indented_code, n=5)
            if args.colour_diff:
                colour_diff(diff)
            else:
                print(''.join(list(diff)))
        elif args.output:
            print(indent_result, end='')

        if args.modify:
            # write in binary mode to preserve the original line ending
            with open(output_file, 'wb') as indented_file:
                indented_file.write(indent_result.encode('utf8'))


def indent_files(arguments):
    """ indent_files(arguments)

    Note: if the parallel option is provided, the files will be read and processed
    in parallel

    """
    args = parse_args(arguments)
    if not args.files:
        # Indent from stdin
        code = sys.stdin.read()
        indent_result = indent_code(code, args)
        _post_indentation(indent_result)

    if args.parallel:
        import multiprocessing
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        pool.starmap(indent_file, [(fname, args) for fname in args.files])
    else:
        for fname in args.files:
            indent_file(fname, args)


def indent_file(fname, args):
    """
    indent_file(fname: string, args)

    1. Create a backup of the source file(backup_source_file())
    2. Read the file contents(read_file())
    3. Indent the code(indent_code())
    4. Write to the file or print the indented code(_post_indentation())

    """
    args = parse_args(args)
    fname = os.path.expanduser(fname)
    code = read_file(fname)
    if not args.dialect:
        # Guess dialect from the file extensions if none is specified in the command
        # line
        if fname.endswith('.lisp'):
            args.dialect = 'lisp'
        elif fname.endswith('.lsp'):
            args.dialect = 'newlisp'
        elif re.search(".clj[sc]{0,1}$", fname):
            args.dialect = 'clojure'
        elif fname.endswith('.ss') or fname.endswith('.scm'):
            args.dialect = 'scheme'
        else:
            args.dialect = 'all'
    indent_result = indent_code(code, args)

    if args.backup:
        # Create a backup file in the specified directory
        backup_source_file(fname, args)

    _post_indentation(indent_result, fpath=fname)


def main():
    """ Entry point """
    indent_files(sys.argv[1:])


if __name__ == '__main__':
    main()