work

.ruff.toml

@@ -24,6 +24,7 @@ ignore = [
   'E711',
   'E712',
   'E741',
+  'F401', # unused imports
   'F841',
   'PT004',
   'SIM102',

init-server.py

@@ -30,8 +30,6 @@ def prepare_shared(c):
 
 def prepare_tile_gen(c):
     install_planetiler(c)
-    apt_get_install(c, 'btrfs-progs')
-
     c.sudo('chown -R ofm:ofm /data/ofm')
 
     for file in [
@@ -40,8 +38,6 @@ def prepare_tile_gen(c):
         'planetiler_planet.sh',
         'prepare-virtualenv.sh',
         'upload_cloudflare.sh',
-        'extract_mbtiles/extract_mbtiles.py',
-        'shrink_btrfs/extract_mbtiles.py',
     ]:
         put(
             c,
@@ -54,19 +50,19 @@ def prepare_tile_gen(c):
     put(
         c,
         scripts / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py',
-        TILE_GEN_BIN / 'extract_mbtiles',
+        f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py',
         permissions='755',
         owner='ofm',
-        target_is_dir=True,
+        create_parent_dir=True,
     )
 
     put(
         c,
         scripts / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py',
-        TILE_GEN_BIN / 'shrink_btrfs',
+        f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py',
         permissions='755',
         owner='ofm',
-        target_is_dir=True,
+        create_parent_dir=True,
     )
 
     sudo_cmd(c, f'cd {TILE_GEN_BIN} && source prepare-virtualenv.sh', user='ofm')

scripts/tile_gen/extract_btrfs.sh

@@ -92,4 +92,4 @@ mv image2.btrfs tiles.btrfs
 
 pigz tiles.btrfs --fast
 
-echo DONE
+echo extract_btrfs.sh DONE

scripts/tile_gen/extract_mbtiles/extract_mbtiles.py

@@ -42,7 +42,7 @@ def cli(mbtiles_path: Path, dir_path: Path):
     #     print(f'Tile number: {calculate_tiles_sum(14)} - OK')
 
     write_metadata(c, dir_path=dir_path)
-    print('DONE')
+    print('extract_mbtiles.py DONE')
 
 
 def write_metadata(c, *, dir_path):

scripts/tile_gen/planetiler_monaco.sh

@@ -22,8 +22,8 @@ java -Xmx1g \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
-  > "planetiler_out.log" 2> "planetiler_err.log"
+  > planetiler_out 2> planetiler_err
 
 rm -r data
 
-$TILE_GEN_BIN/extract_btrfs.sh
+#$TILE_GEN_BIN/extract_btrfs.sh

scripts/tile_gen/planetiler_planet.sh

@@ -24,7 +24,8 @@ java -Xmx30g \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
-  > "planetiler_out.log" 2> "planetiler_err.log"
+  > planetiler_out 2> planetiler_err
 
 rm -r data
 
+#$TILE_GEN_BIN/extract_btrfs.sh

scripts/tile_gen/prepare-virtualenv.sh

@@ -11,5 +11,6 @@ venv/bin/pip -V
 
 venv/bin/pip install -U pip wheel setuptools
 
+venv/bin/pip install click
 
 

scripts/tile_gen/shrink_btrfs/shrink_btrfs.py

@@ -70,6 +70,7 @@ def cli(btrfs_img: Path):
 
     subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
     print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
+    print('shrink_btrfs.py DONE')
 
 
 def get_usage(mnt: Path, key: str):

ssh_lib/pkg_base.py

@@ -7,40 +7,16 @@
 )
 
 
-def pkg_upgrade(c):
-    apt_get_update(c)
-    c.sudo(
-        'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"'
-    )
-
-
-def pkg_clean(c):
-    pkg_list = [
-        # firewalls
-        'ufw',
-        'nftables',
-        'firewalld',
-        'iptables-persistent',
-        # bloat
-        'ntfs-3g',
-        'popularity-contest',
-        'landscape*',
-        'ubuntu-advantage-tools',
-    ]
-
-    apt_get_purge(c, ' '.join(pkg_list))
-    apt_get_autoremove(c)
-    sudo_cmd(c, 'dpkg --list | grep "^rc" | cut -d " " -f 3 | xargs -r dpkg --purge')
-    c.sudo('iptables -L', warn=True)
-
-
 def pkg_base(c):
     pkg_list = [
         'lsb-release',
         'wget',
         'git',
         'build-essential',
         'unzip',
+        'rsync',
+        'btrfs-progs',
+        'pigz',
         #
         'gnupg2',
         'gnupg-agent',
@@ -56,3 +32,30 @@ def pkg_base(c):
     ]
 
     apt_get_install(c, ' '.join(pkg_list))
+
+
+def pkg_clean(c):
+    pkg_list = [
+        # firewalls
+        'ufw',
+        'nftables',
+        'firewalld',
+        'iptables-persistent',
+        # bloat
+        'ntfs-3g',
+        'popularity-contest',
+        'landscape*',
+        'ubuntu-advantage-tools',
+    ]
+
+    apt_get_purge(c, ' '.join(pkg_list))
+    apt_get_autoremove(c)
+    sudo_cmd(c, 'dpkg --list | grep "^rc" | cut -d " " -f 3 | xargs -r dpkg --purge')
+    c.sudo('iptables -L', warn=True)
+
+
+def pkg_upgrade(c):
+    apt_get_update(c)
+    c.sudo(
+        'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"'
+    )

ssh_lib/utils.py

@@ -4,12 +4,17 @@
 
 
 def put(
-    c, local_path, remote_path, permissions=None, owner='root', group=None, target_is_dir=False
+    c, local_path, remote_path, permissions=None, owner='root', group=None, create_parent_dir=False
 ):
     tmp_path = f'/tmp/fabtmp_{random_string(8)}'
     c.put(local_path, tmp_path)
 
-    if is_dir(c, remote_path) or target_is_dir:
+    if create_parent_dir:
+        dirname = os.path.dirname(remote_path)
+        c.sudo(f'mkdir -p {dirname}')
+        set_permission(c, dirname, owner=owner, group=group)
+
+    if is_dir(c, remote_path):
         if not remote_path.endswith('/'):
             remote_path += '/'