backup_code_laundry_incremental.sh

#!/bin/bash

# Incremental GitHub to Google Drive Backup Script for Code Laundry Repositories
# This script backs up only recently modified repositories from the code-laundry GitHub organization
# to Google Drive in the folder: CodeLaundry/Cloud Backup/GitHub
#
# Features:
# - Only backs up repositories with commits in the last N days (default: 30)
# - Configurable time window via -d flag

set -e

# Configuration
GITHUB_ORG="code-laundry"
GDRIVE_FOLDER="CodeLaundry/Cloud Backups/GitHub"
BACKUP_DIR="${TMPDIR:-/tmp}/code-laundry-backup-$$"
RCLONE_REMOTE="gdrive"
DEFAULT_DAYS=30

# Logging configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}" .sh)"
LOCAL_ZIPS_DIR="${SCRIPT_DIR}/zips"
LOG_DIR="${SCRIPT_DIR}/logs"
LOG_TIMESTAMP="$(date '+%Y-%m-%d_%H-%M-%S')"
LOG_FILE="${LOG_DIR}/${SCRIPT_NAME}_${LOG_TIMESTAMP}.log"

# Create logs and local zips directories if they don't exist
mkdir -p "${LOG_DIR}"
mkdir -p "${LOCAL_ZIPS_DIR}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Log functions write to both console (with colors) and log file (without colors)
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $1" >> "${LOG_FILE}"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [WARN] $1" >> "${LOG_FILE}"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $1" >> "${LOG_FILE}"
}

log_skip() {
    echo -e "${BLUE}[SKIP]${NC} $1"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [SKIP] $1" >> "${LOG_FILE}"
}

# Check if a command exists
command_exists() {
    command -v "$1" >/dev/null 2>&1
}

# Install Homebrew if not present (macOS)
install_homebrew() {
    if ! command_exists brew; then
        log_info "Installing Homebrew..."
        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
    fi
}

# Install GitHub CLI
install_gh() {
    if ! command_exists gh; then
        log_info "Installing GitHub CLI..."
        if [[ "$OSTYPE" == "darwin"* ]]; then
            install_homebrew
            brew install gh
        elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
            echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
            sudo apt update
            sudo apt install gh -y
        else
            log_error "Unsupported OS. Please install GitHub CLI manually: https://cli.github.com/"
            exit 1
        fi
    fi
}

# Install rclone
install_rclone() {
    if ! command_exists rclone; then
        log_info "Installing rclone..."
        if [[ "$OSTYPE" == "darwin"* ]]; then
            install_homebrew
            brew install rclone
        elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl https://rclone.org/install.sh | sudo bash
        else
            log_error "Unsupported OS. Please install rclone manually: https://rclone.org/install/"
            exit 1
        fi
    fi
}

# Check GitHub authentication
check_gh_auth() {
    if ! gh auth status >/dev/null 2>&1; then
        log_info "GitHub CLI not authenticated. Starting authentication..."
        gh auth login
    fi
    log_info "GitHub CLI authenticated."
}

# Check rclone Google Drive configuration
check_rclone_config() {
    if ! rclone listremotes | grep -q "^${RCLONE_REMOTE}:"; then
        log_warn "rclone remote '${RCLONE_REMOTE}' not configured."
        log_info "Starting rclone configuration for Google Drive..."
        echo ""
        echo "Please follow these steps:"
        echo "1. Choose 'n' for new remote"
        echo "2. Name it: gdrive"
        echo "3. Choose 'drive' (Google Drive)"
        echo "4. Leave client_id and client_secret blank (press Enter)"
        echo "5. Choose '1' for full access"
        echo "6. Leave root_folder_id blank"
        echo "7. Leave service_account_file blank"
        echo "8. Choose 'n' for advanced config"
        echo "9. Choose 'y' for auto config (will open browser)"
        echo "10. Choose 'n' for shared drive"
        echo "11. Confirm with 'y'"
        echo "12. Quit with 'q'"
        echo ""
        rclone config
    fi

    # Verify the remote works
    if ! rclone lsd "${RCLONE_REMOTE}:" >/dev/null 2>&1; then
        log_error "Cannot access Google Drive. Please reconfigure rclone."
        rclone config
    fi
    log_info "rclone Google Drive configured."
}

# Create backup directory
setup_backup_dir() {
    log_info "Creating backup directory: ${BACKUP_DIR}"
    mkdir -p "${BACKUP_DIR}"
    cd "${BACKUP_DIR}"
}

# Get all repositories from the organization
get_repositories() {
    log_info "Fetching repository list from ${GITHUB_ORG}..."
    gh repo list "${GITHUB_ORG}" --limit 1000 --json name,isArchived --jq '.[] | select(.isArchived == false) | .name'
}

# Check if repository has commits within the specified number of days (across all branches)
has_recent_commits() {
    local repo_name="$1"
    local days="$2"
    local since_date
    local commit_count
    local branches
    local branch

    # Calculate the date N days ago in ISO 8601 format
    if [[ "$OSTYPE" == "darwin"* ]]; then
        since_date=$(date -v-${days}d -u +"%Y-%m-%dT%H:%M:%SZ")
    else
        since_date=$(date -u -d "${days} days ago" +"%Y-%m-%dT%H:%M:%SZ")
    fi

    # Get all branches for this repository
    branches=$(gh api "repos/${GITHUB_ORG}/${repo_name}/branches" --jq '.[].name' 2>/dev/null || echo "")

    if [[ -z "${branches}" ]]; then
        # Fallback to default branch if we can't get branches
        commit_count=$(gh api "repos/${GITHUB_ORG}/${repo_name}/commits?since=${since_date}&per_page=1" --jq 'length' 2>/dev/null || echo "0")
        if [[ "${commit_count}" -gt 0 ]]; then
            return 0
        fi
        return 1
    fi

    # Check each branch for recent commits
    while IFS= read -r branch; do
        if [[ -n "${branch}" ]]; then
            commit_count=$(gh api "repos/${GITHUB_ORG}/${repo_name}/commits?sha=${branch}&since=${since_date}&per_page=1" --jq 'length' 2>/dev/null || echo "0")
            if [[ "${commit_count}" -gt 0 ]]; then
                return 0  # Has recent commits on this branch
            fi
        fi
    done <<< "${branches}"

    return 1  # No recent commits on any branch
}

# Get the date of the last commit for display purposes (across all branches)
get_last_commit_date() {
    local repo_name="$1"
    local branches
    local branch
    local latest_date=""
    local commit_date

    # Get all branches for this repository
    branches=$(gh api "repos/${GITHUB_ORG}/${repo_name}/branches" --jq '.[].name' 2>/dev/null || echo "")

    if [[ -z "${branches}" ]]; then
        # Fallback to default branch
        gh api "repos/${GITHUB_ORG}/${repo_name}/commits?per_page=1" --jq '.[0].commit.committer.date // empty' 2>/dev/null || echo "unknown"
        return
    fi

    # Check each branch and find the most recent commit
    while IFS= read -r branch; do
        if [[ -n "${branch}" ]]; then
            commit_date=$(gh api "repos/${GITHUB_ORG}/${repo_name}/commits?sha=${branch}&per_page=1" --jq '.[0].commit.committer.date // empty' 2>/dev/null || echo "")
            if [[ -n "${commit_date}" ]]; then
                if [[ -z "${latest_date}" ]] || [[ "${commit_date}" > "${latest_date}" ]]; then
                    latest_date="${commit_date}"
                fi
            fi
        fi
    done <<< "${branches}"

    if [[ -n "${latest_date}" ]]; then
        echo "${latest_date}"
    else
        echo "unknown"
    fi
}

# Clone and zip a repository (all branches)
backup_repository() {
    local repo_name="$1"
    local zip_file="${repo_name}.zip"

    log_info "Processing repository: ${repo_name}"

    # Clone the repository with all branches
    log_info "  Cloning ${repo_name} (all branches)..."
    if ! gh repo clone "${GITHUB_ORG}/${repo_name}" "${repo_name}" -- --quiet 2>/dev/null; then
        log_error "  Failed to clone ${repo_name}"
        return 1
    fi

    # Fetch all remote branches and check them out locally
    log_info "  Fetching all branches..."
    (
        cd "${repo_name}"
        # Fetch all remote branches
        git fetch --all --quiet 2>/dev/null || true
        # Check out each remote branch as a local branch
        for remote_branch in $(git branch -r | grep -v '\->' | grep -v 'HEAD' | sed 's/origin\///'); do
            if [[ "${remote_branch}" != "$(git rev-parse --abbrev-ref HEAD)" ]]; then
                git branch --track "${remote_branch}" "origin/${remote_branch}" 2>/dev/null || true
            fi
        done
    )

    local branch_count
    branch_count=$(cd "${repo_name}" && git branch | wc -l | tr -d ' ')
    log_info "  Found ${branch_count} branch(es)"

    # Create zip file (including .git directory to preserve all branches)
    log_info "  Creating zip archive..."
    zip -rq "${zip_file}" "${repo_name}"

    # Remove cloned directory to save space
    rm -rf "${repo_name}"

    # Save a copy of the zip to the local zips folder
    log_info "  Saving zip to local backups folder..."
    cp "${zip_file}" "${LOCAL_ZIPS_DIR}/${zip_file}"

    # Upload to Google Drive
    log_info "  Uploading to Google Drive..."

    # Check if file already exists and delete it (for replacement)
    if rclone lsf "${RCLONE_REMOTE}:${GDRIVE_FOLDER}/${zip_file}" >/dev/null 2>&1; then
        log_info "  Replacing existing file..."
        rclone deletefile "${RCLONE_REMOTE}:${GDRIVE_FOLDER}/${zip_file}" 2>/dev/null || true
    fi

    # Upload the file
    if rclone copy "${zip_file}" "${RCLONE_REMOTE}:${GDRIVE_FOLDER}/" --progress; then
        log_info "  Successfully uploaded ${zip_file}"
    else
        log_error "  Failed to upload ${zip_file}"
        return 1
    fi

    # Remove local zip file
    rm -f "${zip_file}"

    return 0
}

# Cleanup function
cleanup() {
    log_info "Cleaning up..."
    if [[ -d "${BACKUP_DIR}" ]]; then
        rm -rf "${BACKUP_DIR}"
    fi
}

# Default values
days_threshold=${DEFAULT_DAYS}
force_backup=false

# Parse command line arguments
while getopts "d:fh" opt; do
    case ${opt} in
        d)
            days_threshold="${OPTARG}"
            if ! [[ "${days_threshold}" =~ ^[0-9]+$ ]]; then
                log_error "Invalid number of days: ${days_threshold}"
                exit 1
            fi
            ;;
        f)
            force_backup=true
            ;;
        h)
            echo "Usage: $0 [-d days] [-f] [-h]"
            echo ""
            echo "Options:"
            echo "  -d days  Number of days to look back for commits (default: ${DEFAULT_DAYS})"
            echo "  -f       Force backup all repositories (ignore modification check)"
            echo "  -h       Show this help message"
            echo ""
            echo "Examples:"
            echo "  $0           # Backup repos with commits in last 30 days"
            echo "  $0 -d 7      # Backup repos with commits in last 7 days"
            echo "  $0 -d 90     # Backup repos with commits in last 90 days"
            echo "  $0 -f        # Force backup all repositories"
            exit 0
            ;;
        *)
            echo "Usage: $0 [-d days] [-f] [-h]"
            exit 1
            ;;
    esac
done

# Main execution
main() {
    echo ""
    echo "=========================================="
    echo "  Code Laundry Incremental Backup Script"
    echo "=========================================="
    echo ""
    log_info "Log file: ${LOG_FILE}"
    echo ""

    if [[ "${force_backup}" == "true" ]]; then
        log_warn "Force mode enabled - will backup all repositories"
    else
        log_info "Backing up repositories with commits in the last ${days_threshold} days"
    fi
    echo ""

    # Set trap for cleanup on exit
    trap cleanup EXIT

    # Install and check dependencies
    log_info "Checking dependencies..."
    install_gh
    install_rclone

    # Authenticate and configure
    check_gh_auth
    check_rclone_config

    # Ensure Google Drive folder exists
    log_info "Ensuring Google Drive folder exists: ${GDRIVE_FOLDER}"
    rclone mkdir "${RCLONE_REMOTE}:${GDRIVE_FOLDER}" 2>/dev/null || true

    # Setup backup directory
    setup_backup_dir

    # Get repositories
    repos=$(get_repositories)

    if [[ -z "$repos" ]]; then
        log_error "No repositories found in ${GITHUB_ORG}"
        exit 1
    fi

    # Count repositories
    repo_count=$(echo "$repos" | wc -l | tr -d ' ')
    log_info "Found ${repo_count} repositories to check"
    echo ""

    # Backup each repository
    success_count=0
    fail_count=0
    skip_count=0

    while IFS= read -r repo; do
        if [[ -n "$repo" ]]; then
            echo "----------------------------------------"
            log_info "Checking: ${repo}"

            if [[ "${force_backup}" == "true" ]]; then
                # Force backup - skip modification check
                if backup_repository "$repo"; then
                    ((success_count++))
                else
                    ((fail_count++))
                fi
            else
                # Check if repository has recent commits
                if has_recent_commits "${repo}" "${days_threshold}"; then
                    log_info "  Has commits within the last ${days_threshold} days"
                    if backup_repository "$repo"; then
                        ((success_count++))
                    else
                        ((fail_count++))
                    fi
                else
                    last_commit=$(get_last_commit_date "${repo}")
                    log_skip "${repo} - No commits in last ${days_threshold} days (last: ${last_commit})"
                    ((skip_count++))
                fi
            fi
            echo ""
        fi
    done <<< "$repos"

    # Create master zip of all individual repo zips
    if [[ -d "${LOCAL_ZIPS_DIR}" ]] && [[ -n "$(ls -A "${LOCAL_ZIPS_DIR}" 2>/dev/null)" ]]; then
        MASTER_ZIP="${SCRIPT_DIR}/code-laundry-backup_${LOG_TIMESTAMP}.zip"
        log_info "Creating master zip archive: ${MASTER_ZIP}"
        (cd "${SCRIPT_DIR}" && zip -rq "${MASTER_ZIP}" "zips/")
        rm -rf "${LOCAL_ZIPS_DIR}"
        log_info "Master zip saved: ${MASTER_ZIP}"
    else
        log_warn "No zips found in local folder — master zip not created."
    fi
    echo ""

    # Summary
    echo "=========================================="
    echo "  Backup Complete"
    echo "=========================================="
    echo ""
    log_info "Successfully backed up: ${success_count} repositories"
    log_info "Skipped (no recent commits): ${skip_count} repositories"
    if [[ $fail_count -gt 0 ]]; then
        log_warn "Failed to backup: ${fail_count} repositories"
    fi
    echo ""
    log_info "Backups stored in: Google Drive/${GDRIVE_FOLDER}"
    log_info "Master zip stored in: ${SCRIPT_DIR}"
}

# Run main function
main "$@"