diff --git a/init-server.py b/init-server.py index d810cf4..f74f74c 100755 --- a/init-server.py +++ b/init-server.py @@ -3,7 +3,7 @@ import click from fabric import Config, Connection -from ssh_lib import SCRIPTS_DIR, VENV_BIN, dotenv_val +from ssh_lib import SCRIPTS_DIR, TILE_GEN_BIN, VENV_BIN, dotenv_val from ssh_lib.planetiler import planetiler from ssh_lib.tasks import ( prepare_http_host, @@ -120,7 +120,7 @@ def debug(hostname, user, port): # upload_http_host_files(c) # sudo_cmd(c, f'{VENV_BIN}/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync') - planetiler(c) + put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}') if __name__ == '__main__': diff --git a/scripts/tile_gen/cloudflare_upload.sh b/scripts/tile_gen/cloudflare_upload.sh deleted file mode 100755 index 7826c15..0000000 --- a/scripts/tile_gen/cloudflare_upload.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -e - -AREA=$(basename "$(dirname "$PWD")") -VERSION=$(basename "$PWD") - -export RCLONE_CONFIG=/data/ofm/config/rclone.conf -SCRIPT_DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) - -if [[ $AREA != "planet" && $AREA != "monaco" ]]; then - echo "Area must be 'planet' or 'monaco'. Terminating." - exit 1 -fi - -if [ ! -f $RCLONE_CONFIG ]; then - echo "rclone.conf does not exist. Terminating." - exit 1 -fi - - -mkdir -p logs -rm -f logs/rclone.log - -rclone sync \ - --transfers=8 \ - --multi-thread-streams=8 \ - --fast-list \ - -v \ - --stats-file-name-length 0 \ - --stats-one-line \ - --log-file logs/rclone.log \ - --exclude 'logs/**' \ - . "remote:ofm-$AREA/$VERSION" - - -bash "$SCRIPT_DIR/cloudflare_index.sh" \ No newline at end of file diff --git a/scripts/tile_gen/upload_manager.py b/scripts/tile_gen/upload_manager.py new file mode 100755 index 0000000..b9de98f --- /dev/null +++ b/scripts/tile_gen/upload_manager.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +import json +import pathlib +import subprocess + +import click + + +AREAS = ['planet', 'monaco'] + +RUNS_DIR = pathlib.Path('/data/ofm/tile_gen/runs') + + +def upload_rclone(area, run): + subprocess.run( + [ + 'rclone', + 'sync', + '--transfers=8', + '--multi-thread-streams=8', + '--fast-list', + '-v', + '--stats-file-name-length', + '0', + '--stats-one-line', + '--log-file', + RUNS_DIR / area / run / 'logs' / 'rclone.log', + '--exclude', + 'logs/**', + RUNS_DIR / area / run, + f'remote:ofm-{area}/{run}', + ], + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + check=True, + ) + + +@click.group() +def cli(): + """ + Uploads runs to Cloudflare + """ + + +@cli.command() +def upload_runs(): + """ + Upload all runs present in system + """ + + print('running upload_runs') + + for area in AREAS: + if not (RUNS_DIR / area).exists(): + continue + + p = subprocess.run( + [ + 'rclone', + 'lsjson', + '--dirs-only', + '--fast-list', + f'remote:ofm-{area}', + ], + text=True, + capture_output=True, + check=True, + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + ) + rclone_json = json.loads(p.stdout) + runs_remote = {p['Path'] for p in rclone_json} + runs_local = {p.name for p in (RUNS_DIR / area).iterdir()} + + runs_to_upload = runs_local - runs_remote + for run in runs_to_upload: + print(f'uploading {area} {run}') + upload_rclone(area, run) + + +if __name__ == '__main__': + cli()