downloader start

Author: hyperknot

.gitignore

@@ -10,10 +10,11 @@ rclone.conf
 .DS_Store
 
 node_modules
+tmp
+temp
+
 
-/venv
 /.idea
-/temp
-/rclone-ofm.conf
-/scripts/tile_gen/extract_mbtiles/out
+venv
+
 

config/rclone.conf.sample

@@ -4,4 +4,4 @@ provider = Cloudflare
 access_key_id = xxx
 secret_access_key = xxx
 endpoint = https://xxx.r2.cloudflarestorage.com
-
+no_check_bucket = true

init-server.py

@@ -6,7 +6,14 @@
 from fabric import Config, Connection
 
 from ssh_lib.benchmark import c1000k
-from ssh_lib.config import CONFIG_DIR, OFM_DIR, REMOTE_CONFIG, SCRIPTS_DIR, TILE_GEN_BIN
+from ssh_lib.config import (
+    CONFIG_DIR,
+    HTTP_HOST_BIN,
+    OFM_DIR,
+    REMOTE_CONFIG,
+    SCRIPTS_DIR,
+    TILE_GEN_BIN,
+)
 from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings
 from ssh_lib.nginx import certbot, nginx
 from ssh_lib.pkg_base import pkg_base, pkg_upgrade
@@ -91,9 +98,23 @@ def prepare_tile_gen(c):
 
 
 def prepare_http_host(c):
-    nginx(c)
-    certbot(c)
-    c1000k(c)
+    # nginx(c)
+    # certbot(c)
+    # c1000k(c)
+
+    prepare_venv(c)
+
+    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
+
+    for file in [
+        'downloader.py',
+    ]:
+        put(
+            c,
+            SCRIPTS_DIR / 'http_host' / file,
+            HTTP_HOST_BIN,
+            permissions='755',
+        )
 
 
 def debug_tmp(c):

scripts/http_host/downloader.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+import requests
+
+
+DEFAULT_RUNS_DIR = '/data/ofm/http_host/runs'
+
+
+@click.command()
+@click.option('--area', default='planet', help='The area to process')
+@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
+@click.option(
+    '--runs-dir',
+    help='Specify /runs directory',
+    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
+)
+@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
+def cli(area: str, version: str, list_versions: bool, runs_dir: Path):
+    if area not in {'planet', 'monaco'}:
+        sys.exit('Area must be planet or monaco')
+
+    r = requests.get(f'https://{area}.openfreemap.com/dirs.txt')
+    r.raise_for_status()
+
+    versions = sorted(r.text.splitlines())
+
+    all_versions_str = '\n'.join(versions)
+    if list_versions:
+        print(all_versions_str)
+        return
+
+    if version == 'latest':
+        selected_version = versions[-1]
+    else:
+        if version not in versions:
+            sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
+        selected_version = version
+
+    download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR)
+
+
+def download(area: str, version: str, runs_dir: Path):
+    click.echo(f'Downloading: area: {area}, version: {version}')
+
+    version_dir = runs_dir / version
+    btrfs_file = version_dir / 'tiles.btrfs'
+    if btrfs_file.exists():
+        print('File exists, skipping download')
+        return
+
+    temp_dir = runs_dir / '_tmp'
+    shutil.rmtree(temp_dir, ignore_errors=True)
+    temp_dir.mkdir(parents=True)
+
+    gzip_file = temp_dir / 'tiles.btrfs.gz'
+
+    url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
+    print(url)
+
+    subprocess.run(
+        [
+            'aria2c',
+            '--split=8',
+            '--max-connection-per-server=8',
+            '--file-allocation=none',
+            '-o',
+            gzip_file,
+            url,
+        ],
+        check=True,
+    )
+
+    subprocess.run(['unpigz', gzip_file])
+    btrfs_src = temp_dir / 'tiles.btrfs'
+
+    version_dir.mkdir()
+    btrfs_src.rename(btrfs_file)
+
+    shutil.rmtree(temp_dir)
+
+
+if __name__ == '__main__':
+    cli()

scripts/http_host/downloader/downloader.py

@@ -11,6 +11,6 @@ venv/bin/pip -V
 
 venv/bin/pip install -U pip wheel setuptools
 
-venv/bin/pip install click
+venv/bin/pip install click requests
 
 

scripts/prepare-virtualenv.sh

@@ -7,6 +7,7 @@
     'python-dotenv',
     'click',
     'nginxfmt',
+    'requests',
 ]
 
 setup(

setup.py

@@ -11,3 +11,4 @@
 OFM_DIR = '/data/ofm'
 REMOTE_CONFIG = '/data/ofm/config'
 TILE_GEN_BIN = '/data/ofm/tile_gen/bin'
+HTTP_HOST_BIN = '/data/ofm/http_host/bin'