git-sym

#!/usr/bin/env python2.7
"""git-sym

Copyright (c) 2015 Christopher Dunn

https://github.com/cdunn2001/git-sym

MIT LICENSE
------------------------------------
Implementation deailts
======================
## Environment variables

    GIT_SYM_CACHE_DIR

To see the current value or default:

    git-sym --help

## Symlink indirection

We use 3 levels of symlink.
1. The first are committed in the git repo.
2. They point thru GIT_ROOT/.git-sym
3. That points to GIT_DIR/git-sym/links
4. Those point to GIT_SYM_CACHE_DIR

If cache-dir changes, old links are *not* updated unless broken.

## Changes in .git/config
We expect this in .gitconfig:

    [alias]
    gsexec = "!exec "

That facilitates running this script, and is also used within this script.
(Git aliases are always run in the root of a Git tree.)
  http://stackoverflow.com/questions/957928/is-there-a-way-to-get-the-git-root-directory-in-one-command

If missing, we add that to .git/config instead.
"""
from contextlib import contextmanager
import sys, os, re, subprocess, argparse, traceback, ConfigParser, shutil


def log_msg(msg):
    sys.stderr.write(str(msg))
    sys.stderr.write('\n')
def debug_msg(msg):
    sys.stderr.write('#')
    sys.stderr.write(str(msg))
    sys.stderr.write('\n')
def noop(msg):
    pass

debug = noop
log = log_msg


@contextmanager
def cd(newdir, log=log):
    prevdir = os.getcwd()
    os.chdir(os.path.expanduser(newdir))
    log('-> in dir %r' %newdir)
    yield
    os.chdir(prevdir)
    log('<- back to dir %r' %prevdir)
def shell(cmd, log=log):
    """Return result of 'cmd' in a shell.
    cmd: a list of strings
    """
    log("!%s" %cmd)
    return subprocess.check_output(cmd, shell=True) #TODO: Allow python2.6?
def system(cmd):
    """Run cmd as a system-call.
    Raise Exception on any error.
    """
    log(cmd)
    rc = os.system(cmd)
    if rc:
        raise Exception('%d <- %r' %(rc, cmd))
def make_dirs(d):
    debug("makedirs(%s)" %d)
    if not os.path.exists(d):
        log("mkdir -p %s" %d)
        os.makedirs(d)
def rmtree(p):
    if os.path.isdir(p):
        log('Removing directory %r')
        shutil.rmtree(p)
    elif os.path.isfile(p):
        log('Removing file %r')
        os.unlink(p)
def is_in_gitignore(path):
    """Must be relative to CWD, *not* absolute.
    """
    cmd = "git check-ignore -q %r" %path
    try:
        shell(cmd, log=debug)
        return True
    except Exception:
        return False
def redo_GIT_SYM_LINK(GIT_SYM_LINK):
    """GIT_SYM_LINK should *not* be under revision-control.
    However, even if in .gitignore, 'git clean' can remove it.
    So sometimes we need to re-create it.
    We re-create it if broken.
    """
    link_to = os.path.relpath(os.path.join(GIT_SYM_DIR, 'links'),
            start=GIT_ROOT_DIR)
    if (os.path.lexists(GIT_SYM_LINK) and
            os.readlink(GIT_SYM_LINK) != link_to):
        #debug('%r != %r'%(os.readlink(GIT_SYM_LINK), link_to))
        os.remove(GIT_SYM_LINK)
    if not os.path.lexists(GIT_SYM_LINK):
        log("os.symlink(%r, %r)" %(link_to, GIT_SYM_LINK))
        os.symlink(link_to, GIT_SYM_LINK)
def set_GIT_SYM_LINK(val):
    """
    Create symlink and add to git-ignores, if needed.
    Needs GIT_DIR and GIT_ROOT_DIR.
    """
    global GIT_SYM_LINK
    GIT_SYM_LINK = val
    # Re-create, in case of 'git clean -x'.
    redo_GIT_SYM_LINK(GIT_SYM_LINK)
    if not is_in_gitignore(os.path.relpath(GIT_SYM_LINK, start=os.getcwd())):
        cmd = "echo '/.git-sym' >> %s/info/exclude" %(
                GIT_DIR)
        shell(cmd, log=debug)
    return GIT_SYM_LINK
def set_GIT_SYM_CACHE_DIR(val):
    global GIT_SYM_CACHE_DIR
    GIT_SYM_CACHE_DIR = os.path.abspath(val)
    if not os.path.isdir(GIT_SYM_CACHE_DIR):
        make_dirs(GIT_SYM_CACHE_DIR)
def get_GIT_DIR():
    """Can return None.
    """
    try:
        # '.git' could be a file for git-submodule, so rely on rev-parse.
        result = shell('git rev-parse --git-dir', log=debug).strip()
    except Exception as e:
        log(e)
        # Assume we are running in the '.git' dir already, as a git-hook.
        result = None
    return result
def get_GIT_ROOT_DIR():
    shell("git config alias.gsexec '!exec '", log=debug)
    result = shell('git gsexec pwd -P', log=debug).strip()
    return result
def read_cfg(root):
    cfg = {
        'symlinks': {},
    }
    filename = os.path.join(GIT_ROOT_DIR, '.git-sym.cfg')
    if os.path.exists(filename):
        parser = ConfigParser.ConfigParser()
        log("filename=%r" %filename)
        from_file = parser.read(filename)
        log("from_file=%r" %from_file)
        cfg.update(from_file)
    debug("cfg=%r" %(cfg,))
    return cfg
def global_setup():
    """We need to know the top-level directory, unless
    we already have some environment variables.
    """
    global GIT_DIR, GIT_ROOT_DIR, GIT_SYM_DIR
    GIT_DIR = os.path.abspath(get_GIT_DIR())
    GIT_SYM_DIR = os.path.join(GIT_DIR, 'git-sym-local')
    GIT_ROOT_DIR = get_GIT_ROOT_DIR()
    debug("GIT_ROOT_DIR=%r" %GIT_ROOT_DIR)
    debug("GIT_DIR=%r" %GIT_DIR)
    debug("GIT_SYM_DIR=%r" %GIT_SYM_DIR)
    #GIT_SYM_INI = read_cfg(GIT_ROOT_DIR)
    make_dirs(os.path.join(GIT_SYM_DIR, 'links'))
def is_link_thru(via, symlink):
    """
    Reject symlink if it does not point to a relative path.
    For simplicity, 'via' should be abspath already.
    """
    if not os.path.islink(symlink):
        return False
    rel_link_to = os.readlink(symlink)
    if os.path.isabs(rel_link_to):
        return False
    link_to = os.path.join(os.path.dirname(symlink), rel_link_to)
    cp = os.path.commonprefix([os.path.abspath(via), os.path.abspath(link_to)])
    return cp == via
def find_all_symlinks():
    """Find all symlinks in git objects beneath the current directory.
    """
    # By using git-ls-tree intead of os.walk(), we do not recurse into submodules.
    # However, we are really looking only at the commit, not the index nor working-dir.
    cmd = 'git ls-tree -r --full-name HEAD'
    ls_tree = shell(cmd, log=debug)
    re_ls_tree = re.compile(r'^120000\s+blob\s+\S+\s+(\S+)$', flags=re.MULTILINE)
    for full_name in re_ls_tree.findall(ls_tree):
        # full_name is from root-dir, not CWD.
        abs_name = os.path.join(GIT_ROOT_DIR, full_name)
        yield abs_name
def normal_filter(abspath):
    return is_link_thru(GIT_SYM_LINK, abspath)
def find_symlinks_if(filter, paths):
    if not paths:
        paths = ['.']
    result = []
    for path in paths:
        if not os.path.islink(path) and os.path.isdir(path):
            found = list(filter_symlinks(filter, path))
            debug('dir: %r %r' %(path, found))
            result.extend(found)
        else:
            # Assume caller wants this.
            result.append(path)
            debug('keep: %r' %path)
    return result
def filter_symlinks(filter, within='.'):
    """Keep symlinks which satisfy 'filter'.
    Ignore absolute symlinks.
    Return both file and dir symlinks.
    """
    debug("filter_symlinks(within=%r)" %within)
    cwd = os.getcwd()
    with cd(within):
        all_abs = list(find_all_symlinks())
    for abs_name in all_abs:
        if filter(abs_name):
            symlink = os.path.relpath(abs_name, start=cwd)
            log('symlink: %r' %symlink)
            yield symlink
def show_symlinks(abs_symlinks, via):
    """For each symlink,
    describe how we interpret it.

    '+' Unresolved, possibly uncached.
    '.' Cached and fully resolved file.
    '/' Cached and fully resolved directory.
    'O' Ignored.
    """
    cwd = os.getcwd()
    for abs_name in abs_symlinks:
        symlink = os.path.relpath(abs_name, start=cwd)
        if not os.path.islink(abs_name):
            line = '? %s\n' %symlink
            sys.stdout.write(line)
            continue
        elif not is_link_thru(via, abs_name):
            sym = 'O'
        elif os.path.isfile(abs_name):
            sym = '.'
        elif os.path.isdir(abs_name):
            sym = '/'
        else:
            sym = '+'
        link_to = os.readlink(symlink)
        joined_link_to = os.path.normpath(os.path.join(os.path.dirname(symlink), link_to)) \
                if not os.path.isabs(link_to) else link_to
        line = '%s %s\t%s\n' %(sym, symlink, joined_link_to)
        sys.stdout.write(line)
def retrieve_using_make(makefilename, paths):
    MAX_ARG_LEN = 1000
    paths = list(paths) # since we will modify
    while paths:
        a_few = list()
        while len(' '.join(a_few)) < MAX_ARG_LEN and paths:
            a_few.append(paths.pop())
        if len(a_few) > 1:
            # Put one back to avoid exceeding our limit.
            paths.append(a_few.pop())
        silence = '--silent' if log==noop else ''
        cmd = "make -j %s -f %s %s" %(
            silence,
            makefilename,
            ' '.join("'%s'"%p for p in a_few))
        system(cmd)
def retrieve(paths):
    debug("retrieve: %r" %paths)
    if not paths:
        return # to avoid the default make rule
    makefilename = os.path.join(GIT_ROOT_DIR, 'git-sym.makefile')
    with cd(GIT_SYM_CACHE_DIR, log=log):
        retrieve_using_make(makefilename, paths)
    if (os.path.commonprefix([GIT_SYM_CACHE_DIR, GIT_SYM_DIR]) ==
            GIT_SYM_DIR):
        cache_dir = os.path.relpath(GIT_SYM_CACHE_DIR)
    else:
        cache_dir = GIT_SYM_CACHE_DIR
    for path in paths:
        cached_path = os.path.join(cache_dir, path)
        debug('Checking %r -> %r' %(path, cached_path))
        assert os.path.exists(cached_path), cached_path
        if not os.path.exists(path):
            if os.path.lexists(path):
                os.remove(path)
            log("os.symlink(%r, %r)" %(cached_path, path))
            os.symlink(cached_path, path)
        # TODO: Remove these extra checks, unless debug mode?
        assert os.path.exists(path), path
        assert os.path.samefile(cached_path, path), "%r != %r" %(
                cached_path, path)
def get_linked_path(symlink, via):
    linked = os.readlink(symlink)
    norm_linked = os.path.normpath(os.path.join(os.path.dirname(symlink), linked))
    canon_linked = os.path.relpath(norm_linked, start=via)
    debug("%r -> %r [%r] (%r)" %(symlink, linked, canon_linked, norm_linked))
    return canon_linked
def check_link(symlink):
    """
    Return True iff the symlink is resolved.
    """
    if not os.path.isfile(symlink) and not os.path.isdir(symlink):
        log('%r -> %r does not exist' %(symlink, os.readlink(symlink)))
        return False
    return True
def fix(symlink, via_old, via_new):
    assert os.path.isabs(via_old), via_old
    assert os.path.isabs(via_new), via_new
    to_link_old = os.readlink(symlink)
    #debug(to_link_old)
    to_link_joined_old = os.path.join(os.path.dirname(symlink), to_link_old)
    #debug(to_link_joined_old)
    to_link_rel_old = os.path.relpath(to_link_joined_old, start=via_old)
    #debug(to_link_rel_old)
    to_link_joined_new = os.path.join(via_new, to_link_rel_old)
    #debug(to_link_joined_new)
    to_link_new = os.path.relpath(to_link_joined_new, start=os.path.dirname(symlink))
    #debug(to_link_new)
    debug('%r becomes %r' %(to_link_old, to_link_new))
    os.unlink(symlink)
    os.symlink(to_link_new, symlink)
def unique_name(path):
    if os.path.isdir(path):
        return 'dir.' + path
    if os.path.islink(path):
        raise Exception('We do not cache symlinks: %r' %path)
    if not os.path.isfile(path):
        raise Exception('We cannot cache that which does not exist.' %path)
    sha1 = shell('git hash-object %r' %path)
    debug('%r' %sha1)
    sha1 = sha1.strip()
    return 'sha1.' + sha1 + '.' + os.path.basename(path)

def git_sym_update(symlinks, **args):
    symlinks = list(find_symlinks_if(normal_filter, symlinks))
    needed = set()
    for symlink in symlinks:
        if not check_link(symlink):
            basename = os.path.basename(os.readlink(symlink))
            path = get_linked_path(symlink, GIT_SYM_LINK)
            # We require flat, unique link-names within GIT_SYM_DIR/links.
            assert path == basename, (path, basename)
            needed.add(basename)
    debug("needed: %s" %repr(needed))
    with cd(os.path.join(GIT_SYM_DIR, 'links'), log=log):
        retrieve(needed)
    git_sym_check(symlinks)
    # TODO Print on failure, unless silent mode.
def git_sym_link(paths, add=False, diff=False, **args):
    needed = set()
    for path in paths:
        uname = unique_name(path)
        needed.add(uname)
        cached = os.path.join(GIT_SYM_CACHE_DIR, uname)
        if os.path.exists(cached):
            if diff:
                shell('diff -ru %r %r' %(path, cached))
            log('%r already exists, but matches. Removing %r.')
            rmtree(path)
        else:
            shell('mv %s %s' %(path, cached))
        link_to = os.path.relpath(os.path.join(GIT_SYM_LINK, uname), start=os.path.dirname(path))
        log('os.symlink(%r, %r)' %(link_to, path))
        os.symlink(link_to, path)
        if add:
            shell('git add %s' %path, log=log)
            mode = os.stat(cached).st_mode & ~0222
            log('os.chmod(%r, %0o)' %(cached, mode))
            os.chmod(cached, mode)
    # TODO: Test these next 2 lines.
    git_sym_update(paths)
    git_sym_show(paths)
def git_sym_unlink(symlinks, cp, add=False, **args):
    if any(symlink.endswith('/') for symlink in symlinks):
        raise Exception('For "unlink", paths must not end in "/".')
    for symlink in symlinks:
        cached = os.path.realpath(symlink)
        if not os.path.exists(cached):
            raise Exception('%r -> %r does not exist!' %(
                symlink, cached))
        os.unlink(symlink)
        shell('%s %s %s' %(cp, cached, symlink), log=log)
        if add:
            shell('git add %s' %symlink, log=log)
        # TODO: maybe check sha1
def git_sym_show(symlinks, **args):
    filter = lambda abspath: True
    symlinks = find_symlinks_if(filter, symlinks)
    show_symlinks(symlinks, via=GIT_SYM_LINK)
def git_sym_check(symlinks, **args):
    symlinks = list(find_symlinks_if(normal_filter, symlinks))
    for symlink in symlinks:
        if not check_link(symlink):
            raise Exception(symlink)
def git_sym_missing(symlinks, **arsg):
    symlinks = list(find_symlinks_if(normal_filter, symlinks))
    missing = 0
    for symlink in symlinks:
        if not check_link(symlink):
            sys.stdout.write('%s\n' %symlink)
            missing += 1
    #if missing:
    #    raise Exception(missing)
def git_sym_clean(symlinks, **args):
    symlinks = find_symlinks_if(normal_filter, symlinks)
    debug('clean links from %r' %symlinks)
    basenames = [os.path.basename(os.readlink(symlink)) for symlink in symlinks]
    with cd(os.path.join(GIT_SYM_DIR, 'links'), log=log):
        shell('rm -f %s' %' '.join(basenames), log=log)
def git_sym_fix(symlinks, old_link, **args):
    GIT_SYM_VIA_OLD = os.path.abspath(old_link)
    log('via=%r' %GIT_SYM_VIA_OLD)
    def filter(abspath):
        return is_link_thru(GIT_SYM_VIA_OLD, abspath)
    symlinks = list(find_symlinks_if(filter, symlinks))
    log(symlinks)
    for sl in symlinks:
        fix(sl, GIT_SYM_VIA_OLD, GIT_SYM_LINK)
def main(args):
    global log, debug
    if args['silent']:
        log = noop
    if args['debug']:
        debug = debug_msg
    debug(args)
    set_GIT_SYM_LINK(os.path.abspath(args['link']))
    set_GIT_SYM_CACHE_DIR(os.path.abspath(args['cache_dir']))
    debug('GIT_SYM_LINK=%r' %GIT_SYM_LINK)
    debug("GIT_SYM_CACHE_DIR=%r" %GIT_SYM_CACHE_DIR)
    cmd_table = {
            'add': git_sym_link, # deprecated
            'check': git_sym_check,
            'clean': git_sym_clean,
            'fix': git_sym_fix,
            'link': git_sym_link,
            'missing': git_sym_missing,
            'show': git_sym_show,
            'unlink': git_sym_unlink,
            'update': git_sym_update,
    }
    cmd = args['command']
    del args['command']
    try:
        cmd_table[cmd](**args)
    except subprocess.CalledProcessError as e:
        log(e)
        log(" in directory %r" %os.getcwd())
        sys.exit(1)
    except Exception:
        log(traceback.format_exc())
        sys.exit(1)
def parse_args():
    default_GIT_SYM_LINK = os.path.join(GIT_ROOT_DIR, '.git-sym')
    default_GIT_SYM_CACHE_DIR = os.environ.get('GIT_SYM_CACHE_DIR', None)
    if not default_GIT_SYM_CACHE_DIR:
        default_GIT_SYM_CACHE_DIR = os.path.join(GIT_SYM_DIR, 'cache')
    epilog = 'See git-sym in GitHub for details.'
    parser = argparse.ArgumentParser(
            description='Cache symlinks (presumably for large files).',
            epilog=epilog)
    parser.add_argument('--retriever',
            help="(not implemented yet) For now, we always use 'make -f git-sym.makefile'.")
    parser.add_argument('--cache-dir',
            default=default_GIT_SYM_CACHE_DIR,
            help='Directory in which to store retrieved files/directories. [default=%(default)r]')
    parser.add_argument('--link',
            default=default_GIT_SYM_LINK,
            help='Symlink to GIT_SYM_DIR/links. (Intermediate level of indirection.) This path should be ignored by git. [default=%(default)r]')
    parser.add_argument('--silent', '-s', action='store_true',
            help='Otherwise, log to stderr.')
    parser.add_argument('--debug', '-g', action='store_true')

    subs = parser.add_subparsers(dest='command')

    parser_show = subs.add_parser('show',
            help='Show symlinks on stdout. "O" indicates that git-sym will ignore it; "+" that it needs to be resolved; and "." or "/" that it is a fully resolved file or directory.')
    parser_show.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks.')

    parser_update = subs.add_parser('update',
            help='Fill-in symlinks and retrieve files into cache.')
    parser_update.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks.')

    parser_check = subs.add_parser('check',
            help='Look for the first unresolved symlink, if any. Return 1 unless all symlinks are resolved. (Intended for scripting. Humans should use "missing".')
    parser_check.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks.')

    parser_missing = subs.add_parser('missing',
            help='Print all unresolved symlinks on stdout.')
    parser_missing.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks.')

    parser_link = subs.add_parser('link',
            help='Move (mv) named files/directories to GIT_SYM_CACHE_DIR. Create git-sym symlinks in their places. Move one at-a-time in case of a failure.')
    parser_link.add_argument('--mv',
            default='mv -n',
            help='Command to run. [default=%(default)r]')
    parser_link.add_argument('--add', action='store_true',
            help='Also run "git add". You still need to run "git commit".')
    parser_link.add_argument('--diff', action='store_true',
            help='If destination exists, diff before removing path.')
    parser_link.add_argument('paths', nargs='+',
            help='Files and/or directories. At least one path must be supplied.')

    parser_add = subs.add_parser('add',
            help='(Deprecated.) Alias for "link".')
    parser_add.add_argument('paths', nargs='+')

    parser_unlink = subs.add_parser('unlink',
            help='Copy files/directories resolved from the given "symlinks", typically residing in GIT_SYM_CACHE_DIR. Remove symlinks before over-writing.')
    parser_unlink.add_argument('--add', action='store_true',
            help='Also run "git add". You still need to run "git commit".')
    parser_unlink.add_argument('--cp',
            default='cp -a',
            help='Command to run. (On Windows, use cygwin.) A good idea: "rsync -av" [default=%(default)r]')
    parser_unlink.add_argument('symlinks', nargs='+',
            help='Files and/or directories. At least one path must be supplied.')

    parser_fix = subs.add_parser('fix',
            help='(Rarely needed.) Change symlinks from OLD_LINK to LINK. `git add` the changes, but do not commit. (This is helpful if you change the LINK.)')
    parser_fix.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks, but with respect to OLD_LINK, not LINK.')
    parser_fix.add_argument('--old-link',
            default=os.path.join(GIT_ROOT_DIR, 'git-sym'),
            help='[default=%(default)r]')

    parser_clean = subs.add_parser('clean',
            help='(Rarely needed.) Remove named or discovered links from GIT_SYM_DIR/links. (To remove them all, use rm.) Subsequent `git-sym update` will re-create those links and also re-run make, which *might* update cached files.')
    parser_clean.add_argument('symlinks', nargs='*',
            help='If not given, walk through tree to find relevant symlinks.')

    return parser.parse_args()


if __name__=='__main__':
    global_setup()
    args = parse_args()
    main(vars(args))