work

init-server.py

@@ -65,6 +65,13 @@ def debug_tmp(c):
         permissions='755',
         owner='ofm',
     )
+    put(
+        c,
+        scripts / 'shrink_btrfs' / 'shrink_btrfs.py',
+        TILE_GEN_BIN,
+        permissions='755',
+        owner='ofm',
+    )
 
     return
     c.sudo('rm -rf /data/ofm/logs')

scripts/extract_mbtiles/extract_mbtiles.py

@@ -35,13 +35,11 @@ def cli(mbtiles_path: Path, dir_path: Path):
     write_dedupl_files(c, dir_path=dir_path)
     write_tile_files(c, dir_path=dir_path)
 
-    # remove dedupl files at the end
-    shutil.rmtree(dir_path / 'dedupl')
-
     # if it's a full planet run,
     # make sure there are exactly the right number of files generated
-    if 'planet' in mbtiles_path.parent.name:
+    if 'planet' in mbtiles_path.resolve().parent.name:
         assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
+        print(f'Tile number: {calculate_tiles_sum(14)} - OK')
 
     print('DONE')
 

scripts/shrink_btrfs/shrink_btrfs.py

@@ -8,7 +8,7 @@
 import click
 
 
-# btrfs cannot shrink smaller than about 256 MiB
+# btrfs cannot shrink smaller than 256 MiB
 SMALLEST_SIZE = 256 * 1024 * 1024
 
 
@@ -49,11 +49,11 @@ def cli(btrfs_img: Path):
 
         free_bytes = get_usage(mnt_dir, 'Device unallocated')
         device_size = get_usage(mnt_dir, 'Device size')
-        shrink_idea = free_bytes * 0.9
+        shrink_idea = free_bytes * 0.7
 
         # workaround for the SMALLEST_SIZE limit
         if device_size - free_bytes < SMALLEST_SIZE:
-            shrink_idea = (device_size - SMALLEST_SIZE) * 0.9
+            shrink_idea = (device_size - SMALLEST_SIZE) * 0.7
 
         # stop if 10 MB left
         if shrink_idea < 10_000_000:

scripts/tile_gen/extract_btrfs.sh

@@ -5,7 +5,7 @@ sudo umount mnt_rw 2> /dev/null || true
 sudo umount mnt_rw2 2> /dev/null || true
 rm -rf mnt_rw* tmp_*
 rm -f *.btrfs *.gz
-rm -f *.log
+rm -f *.log *.txt
 
 # make an empty file that's definitely bigger then the current OSM output
 fallocate -l 200G image.btrfs
@@ -44,7 +44,10 @@ grep fixed extract_out.log > dedupl_fixed.log || true
 
 # Unfortunately, by deleting files from the btrfs partition, the size _grows_.
 # So we need to rsync onto a new partition.
-rsync -avH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log
+rsync -avH \
+  --exclude dedupl \
+  mnt_rw/extract/ mnt_rw2/extract/ \
+  > rsync_out.log 2> rsync_err.log
 
 
 # collect stats
@@ -61,11 +64,11 @@ sudo btrfs filesystem show mnt_rw
 echo -e "\n\nbtrfs filesystem usage"
 sudo btrfs filesystem usage mnt_rw
 
-echo -e "\n\nbtrfs filesystem du -s"
-sudo btrfs filesystem du -s mnt_rw
-
-echo -e "\n\ncompsize -x"
-sudo compsize -x mnt_rw 2> /dev/null || true
+# takes a lot of time, should only be used when debugging
+#echo -e "\n\nbtrfs filesystem du -s"
+#sudo btrfs filesystem du -s mnt_rw
+#echo -e "\n\ncompsize -x"
+#sudo compsize -x mnt_rw 2> /dev/null || true
 } > stats1.txt
 
 {
@@ -81,11 +84,11 @@ sudo btrfs filesystem show mnt_rw2
 echo -e "\n\nbtrfs filesystem usage"
 sudo btrfs filesystem usage mnt_rw2
 
-echo -e "\n\nbtrfs filesystem du -s"
-sudo btrfs filesystem du -s mnt_rw2
-
-echo -e "\n\ncompsize -x"
-sudo compsize -x mnt_rw2 2> /dev/null || true
+# takes a lot of time, should only be used when debugging
+#echo -e "\n\nbtrfs filesystem du -s"
+#sudo btrfs filesystem du -s mnt_rw2
+#echo -e "\n\ncompsize -x"
+#sudo compsize -x mnt_rw2 2> /dev/null || true
 } > stats2.txt