git-archive-all

#! /usr/bin/env python
# coding=utf-8

__version__ = "1.5.2-CustisMod"
# forked from https://github.com/Kentzo/git-archive-all

import sys
from os import path, extsep
from subprocess import Popen, PIPE
from shutil import rmtree
# from pprint import pprint
import re
import time
import tempfile


class GitArchiver(object):
    """
    GitArchiver

    Scan a git repository and export all tracked files, and submodules.
    Checks for .gitattributes files in each directory and uses 'export-ignore'
    pattern entries for ignore files in the archive.

    Automatically detects output format extension: zip, tar, bz2, or gz.
    """

    def __init__(self, prefix='', verbose=False, exclude=True, force_sub=False, extra=None, main_repo=None, revision=None):
        """
        @type prefix:   string
        @param prefix:  Prefix used to prepend all paths in the resulting archive.

        @type verbose:  bool
        @param verbose: Determines verbosity of the output (stdout).

        @type exclude:  bool
        @param exclude: Determines whether archiver should follow rules specified in .gitattributes files.
                        Defaults to True.

        @type force_sub:    bool
        @param force_sub:   Determines whether submodules are initialized and updated before archiving.
                            Defaults to False

        @type extra:    list
        @param extra:   List of extra paths to include in the resulting archive.

        @type main_repo:    string
        @param main_repo:   Absolute path to the main repository. If None, current cwd is used.

        @type revision:    string
        @param revision:   Either SHA-1 or branch of revision to export, default is HEAD.
        """
        if extra is None:
            extra = []

        if main_repo is None:
            main_repo = path.abspath('')

        if revision is None:
            revision = 'HEAD'

        self.prefix = prefix
        self.verbose = verbose
        self.exclude = exclude
        self.extra = extra
        self.force_sub = force_sub
        self.main_repo = main_repo
        self.revision = revision

    def prepare_buffer_repo(self):
        """
        Prepares the buffer repository to export

        """
        # Export full tree of files including every submodule recursively
        self.buffer_repo_path = path.join(tempfile.gettempdir(), str(int(time.time())))
        self.run_shell('git clone --recursive ' + self.main_repo + ' ' + self.buffer_repo_path)
        self.run_shell('git reset --hard ' + self.revision, self.buffer_repo_path)
        if self.verbose:
            print "Initialized temporary repository at {f}".format(f=self.buffer_repo_path)

    def clean_buffer_repo(self):
        """
        Cleans up temp dir

        """
        rmtree(self.buffer_repo_path)

    def create(self, output_file_path, dry_run=False, format=None):
        """
        Creates the archive, written to the given output_file_path

        Type of the archive is determined either by extension of output_file_path or by the format argument.
        Supported formats are: gz, zip, bz2, tar, tgz

        @type output_file_path:     string
        @param output_file_path:    Output file path.

        @type dry_run:  bool
        @param dry_run: Determines whether create should do nothing but print what it would archive.

        @type format:   string
        @param format:  Determines format of the output archive. If None, format is determined from extension.
        """

        self.prepare_buffer_repo()
        files = set()
        if format is None:
            file_name, file_ext = path.splitext(output_file_path)
            format = file_ext[len(extsep):].lower()

        if format == 'zip':
            from zipfile import ZipFile, ZIP_DEFLATED

            output_archive = ZipFile(path.abspath(output_file_path), 'w')
            add = lambda file_path, file_name: output_archive.write(file_path, path.join(self.prefix, file_name), ZIP_DEFLATED)
        elif format in ['tar', 'bz2', 'gz', 'tgz']:
            import tarfile

            if format == 'tar':
                t_mode = 'w'
            elif format == 'tgz':
                t_mode = 'w:gz'
            else:
                t_mode = 'w:{f}'.format(f=format)

            output_archive = tarfile.open(path.abspath(output_file_path), t_mode)
            add = lambda file_path, file_name: output_archive.add(file_path, path.join(self.prefix, file_name))
        else:
            raise RuntimeError("Unknown format: {f}".format(f=format))

        for file_path in self.extra:
            if not dry_run:
                if self.verbose:
                    print "Compressing {f} => {a}...".format(f=file_path, a=path.join(self.prefix, file_path))
                if not file_path in files:
                    add(file_path, file_path)
                    files.add(file_path)
            else:
                print "{f} => {a}".format(f=file_path, a=path.join(self.prefix, file_path))

        for file_path in self.list_files():
            if not dry_run:
                if self.verbose:
                    print "Compressing {f} => {a}...".format(f=path.join(self.main_repo, file_path), a=path.join(self.prefix, file_path))
                if not file_path in files:
                    add(path.join(self.buffer_repo_path, file_path), file_path)
                    files.add(file_path)
            else:
                print "{f} => {a}".format(f=path.join(self.main_repo, file_path), a=path.join(self.prefix, file_path))

        output_archive.close()
        self.clean_buffer_repo()

    def get_path_components(self, repo_abspath, abspath):
        """
        Splits given abspath into components until repo_abspath is reached.

        E.g. if repo_abspath is '/Documents/Hobby/ParaView/' and abspath is
        '/Documents/Hobby/ParaView/Catalyst/Editions/Base/', function will return:
        ['.', 'Catalyst', 'Editions', 'Base']

        First element is always '.' (concrete symbol depends on OS).

        @type repo_abspath:     string
        @param repo_abspath:    Absolute path to the git repository.

        @type abspath:  string
        @param abspath: Absolute path to within repo_abspath.

        @rtype:     list
        @return:    List of path components.
        """
        components = []
        while not path.samefile(abspath, repo_abspath):
            abspath, tail = path.split(abspath)
            if len(tail):
                components.insert(0, tail)
        components.insert(0, path.relpath(repo_abspath, repo_abspath))
        return components

    def get_exclude_patterns(self, repo_abspath, repo_file_paths):
        """
        Returns exclude patterns for a given repo. It looks for .gitattributes files in repo_file_paths.

        Resulting dictionary will contain exclude patterns per path (relative to the repo_abspath).
        E.g. {('.', 'Catalyst', 'Editions', 'Base'), ['Foo*', '*Bar']}

        @type repo_abspath:     string
        @param repo_abspath:    Absolute path to the git repository.

        @type repo_file_paths:  list
        @param repo_file_paths: List of paths relative to the repo_abspath that are under git control.

        @rtype:         dict
        @return:    Dictionary representing exclude patterns.
                    Keys are tuples of strings. Values are lists of strings.
                    Returns None if self.exclude is not set.
        """
        if not self.exclude:
            return None

        def read_attributes(attributes_abspath):
            patterns = []
            if path.isfile(attributes_abspath):
                attributes = open(attributes_abspath, 'r').readlines()
                patterns = []
                for line in attributes:
                    tokens = line.strip().split()
                    if 'export-ignore' in tokens[1:]:
                        patterns.append(tokens[0])
            return patterns

        exclude_patterns = {}

        global_attributes_abspath = self.run_shell('git config --get core.attributesfile', repo_abspath).rstrip()
        exclude_patterns[()] = read_attributes(global_attributes_abspath)

        for attributes_abspath in [path.join(repo_abspath, f) for f in repo_file_paths if f.endswith('.gitattributes')]:
            # Each .gitattributes affects only files within its directory.
            key = tuple(self.get_path_components(repo_abspath, path.dirname(attributes_abspath)))
            exclude_patterns[key] = read_attributes(attributes_abspath)

        local_attributes_abspath = path.join(repo_abspath, '.git', 'info', 'attributes')
        key = tuple(self.get_path_components(repo_abspath, repo_abspath))
        if key in exclude_patterns:
            exclude_patterns[key].extend(read_attributes(local_attributes_abspath))
        else:
            exclude_patterns[key] = read_attributes(local_attributes_abspath)

        return exclude_patterns

    def is_file_excluded(self, repo_abspath, repo_file_path, exclude_patterns):
        """
        Checks whether file at a given path is excluded.

        @type repo_abspath: string
        @param repo_abspath: Absolute path to the git repository.

        @type repo_file_path:   string
        @param repo_file_path:  Path to a file within repo_abspath.

        @type exclude_patterns:     dict
        @param exclude_patterns:    Exclude patterns with format specified for get_exclude_patterns.

        @rtype: bool
        @return: True if file should be excluded. Otherwise False.
        """
        if exclude_patterns is None or not len(exclude_patterns):
            return False

        from fnmatch import fnmatch

        file_name = path.basename(repo_file_path)
        components = self.get_path_components(repo_abspath, path.join(repo_abspath, path.dirname(repo_file_path)))

        is_excluded = False
        # We should check all patterns specified in intermediate directories to the given file.
        # At the end we should also check for the global patterns (key '()' or empty tuple).
        while not is_excluded:
            key = tuple(components)
            if key in exclude_patterns:
                patterns = exclude_patterns[key]
                for p in patterns:
                    if fnmatch(file_name, p) or fnmatch(repo_file_path, p):
                        if self.verbose:
                            print 'Exclude pattern matched {pattern}: {path}'.format(pattern=p, path=repo_file_path)
                        is_excluded = True

            if not len(components):
                break

            components.pop()

        return is_excluded


    def is_file_invalid(self, file_name, repo_file_path, repo_abspath, exclude_patterns):
        if file_name.startswith('.git') or (not path.islink(repo_file_path) and path.isdir(repo_file_path)):
            return True
        else:
            if self.is_file_excluded(repo_abspath, repo_file_path, exclude_patterns):
                return True
            else:
                return False

    def list_files(self, repo_path=''):
        """
        An iterator method that yields a file path relative to main_repo
        for each file that should be included in the archive.
        Skips those that match the exclusion patterns found in
        any discovered .gitattributes files along the way.

        Recurs into submodules as well.

        @type repo_path:    string
        @param repo_path:   Path to the git submodule repository within the main git repository.

        @rtype:     iterator
        @return:    Iterator to traverse files under git control relative to main_repo.
        """

        repo_abspath = path.join(self.buffer_repo_path, repo_path)

        if self.force_sub:
            self.run_shell("git submodule init", repo_abspath)
            self.run_shell("git submodule update", repo_abspath)

        repo_file_paths = self.run_shell('git ls-files --cached --full-name', repo_abspath).splitlines()
        submodule_paths = self.run_shell("git submodule --quiet foreach 'pwd'", repo_abspath).splitlines()

        exclude_patterns = self.get_exclude_patterns(repo_abspath, repo_file_paths)

        # Eliminate redundant submodule references
        for submodule_path in submodule_paths:
            base_submodule_path = path.basename(submodule_path)
            while base_submodule_path in repo_file_paths:
                repo_file_paths.remove(base_submodule_path)

        for repo_file_path in repo_file_paths:
            repo_file_path = repo_file_path.decode('string_escape').strip('"')  # file path relative to current repo
            file_name = path.basename(repo_file_path)

            # Only list symlinks and files that don't start with git.
            if self.is_file_invalid(file_name, repo_file_path, repo_abspath, exclude_patterns):
                continue

            main_repo_file_path = path.join(repo_path, repo_file_path)  # file path relative to the main repo

            # Yield both repo_file_path and main_repo_file_path to preserve structure of the repo.
            yield main_repo_file_path # file path relative to the main repo

        # List files of every submodule.
        for submodule_path in submodule_paths:
            # In order to get output path we need to exclude repository path from submodule_path.
            submodule_path = path.relpath(submodule_path, self.buffer_repo_path)
            for file_path in self.list_files(submodule_path):
                sub_repo_file_path = path.join(path.basename(submodule_path), path.relpath(file_path, submodule_path))
                # FIXME: Take into considerations submodule's .gitignore and stuff
                if not self.is_file_invalid(path.basename(sub_repo_file_path), sub_repo_file_path, repo_abspath, exclude_patterns):
                    yield sub_repo_file_path

    @staticmethod
    def run_shell(cmd, cwd=None):
        """
        Runs shell command and returns the output.

        @type cmd:  string
        @param cmd: Command to be executed.

        @rtype:     string
        @return:    String representing command output.
        """
        return Popen(cmd, shell=True, stdout=PIPE, cwd=cwd).stdout.read()


if __name__ == "__main__":
    from optparse import OptionParser

    parser = OptionParser(usage="usage: %prog [-v] [--repo .] [--rev HEAD] [] [--prefix PREFIX] [--no-exclude] OUTPUT_FILE",
        version="%prog {version}".format(version=__version__))

    parser.add_option('--repo',
        type='string',
        dest='gitrepo',
        help="Specify working git repo.",
        default='.')

    parser.add_option('--rev',
        type='string',
        dest='revision',
        help="Specify revision or branch to export.",
        default='HEAD')

    parser.add_option('--prefix',
        type='string',
        dest='prefix',
        default='',
        help="Prepend PREFIX to each filename in the archive. OUTPUT_FILE name is used by default to avoid tarbomb.")

    parser.add_option('-v', '--verbose',
        action='store_true',
        dest='verbose',
        help='Enable verbose mode.')

    parser.add_option('--no-exclude',
        action='store_false',
        dest='exclude',
        default=True,
        help="Don't read .gitattributes files for patterns containing export-ignore attrib.")

    parser.add_option('--force-submodules',
        action='store_true',
        dest='force_sub',
        help="Force a git submodule init && git submodule update at each level before iterating submodules.")

    parser.add_option('--extra',
        action='append',
        dest='extra',
        default=[],
        help="Any additional files to include in the archive.")
    parser.add_option('--dry-run',
        action='store_true',
        dest='dry_run',
        help="Don't actually archive anything, just show what would be done.")

    options, args = parser.parse_args()

    if len(args) != 1:
        parser.error('You must specify exactly one output file')

    output_file_path = args[0]

    if path.isdir(output_file_path):
        parser.error('You cannot use directory as output')

    # avoid tarbomb
    if options.prefix:
        options.prefix = path.join(options.prefix, '')
    else:
        import re
        output_name = path.basename(output_file_path)
        output_name = re.sub('(\.zip|\.tar|\.tgz|\.gz|\.bz2|\.tar\.gz|\.tar\.bz2)$', '', output_name) or 'Archive'
        options.prefix = path.join(output_name, '')

    archiver = GitArchiver(options.prefix,
                           options.verbose,
                           options.exclude,
                           options.force_sub,
                           options.extra,
                           options.gitrepo,
                           options.revision)

    try:
        archiver.create(output_file_path, options.dry_run)
    except Exception, e:
        parser.exit(2, "{exception}\n".format(exception=e))

    sys.exit(0)