Skip to content

Commit 05df7d9

Browse files
author
GMan999
committed
more files. cleanup is next step
1 parent 79fa6b3 commit 05df7d9

File tree

10 files changed

+289
-0
lines changed

10 files changed

+289
-0
lines changed

‎CHECKSUMS

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
SHA256 (zip_site.sh) = 56be7f55269775d318714687ace4d50868c8669c459fdd7c4863c9afe26659c5
2+
SHA256 (cull_bad.sh) = 8b529c4328877ade4a2d1ef7e0c98b263aea22b4e60e9ee6c55691cac10608ea
3+
SHA256 (mirror_check.sh) = 915c335eedd7b65c1c1aee59bd3dcd98f043f2a1434efc32b5b47c4856f09936
4+
SHA256 (zip_site.sh) = 56be7f55269775d318714687ace4d50868c8669c459fdd7c4863c9afe26659c5
5+
SHA256 (cull_bad.sh) = 8b529c4328877ade4a2d1ef7e0c98b263aea22b4e60e9ee6c55691cac10608ea
6+
SHA256 (mirror_check.sh) = 915c335eedd7b65c1c1aee59bd3dcd98f043f2a1434efc32b5b47c4856f09936
7+
SHA256 (mirror_pull.sh) = 45b8b691685a6bc5d4a96a9663ba17bf4233ebc371ff2fd8fcdd3c7625b08a9d
8+
SHA256 (tar_site.sh) = 211337a86ae02711237a4a60ba53a0492861ebc4dd505bc7ac39c576b9b08f04
9+
SHA256 (zip_site.sh) = 56be7f55269775d318714687ace4d50868c8669c459fdd7c4863c9afe26659c5

‎README.md.old

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# repl-kit
2+
repl-kit is a suite of shell scripts for maintaining static mirrors for software distribution. repl-kit aims to be a tool for both the master and mirror sites. The master site can confirm the integrity of the offered mirrors, and list them in a simple HTML frame when verified. The mirror sites can update the local software repository as needed. repl-kit (should be) fully portable to any free Unix-like operating system, and is also capable of dwelling in the world of Tor hidden services.

‎cull_bad.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh
2+
# cull_site.sh 20140711 gman
3+
# DEPS none
4+
# cut sites older than 30 days in the bad_sites.txt file
5+
6+
7+
# variables
8+
bad_list="${bad_list:-/tmp/bad_sites.txt}"
9+
10+
# function
11+
12+
some sed or another >30 days
13+
14+

‎generate_checksums.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
3+
touch CHECKSUMS;
4+
5+
/bin/sha256 cull_bad.sh mirror_check.sh mirror_pull.sh tar_site.sh zip_site.sh >>CHECKSUMS

‎mirror_check.sh

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/sh
2+
# check_mirror.sh 20140701 gman
3+
# DEPS wget, tor/torsocks
4+
# determine integrity of AroundBlocks.info mirror compared to local copy
5+
# add to iframe_output.html iframe if good, add to bad_sites if bad
6+
# to use add full urls to $urls file. periodic housekeeping of it is fine.
7+
# Mirror site data is stored in /tmp since it can get sloppy.
8+
9+
# variables
10+
11+
store="${store:-/tmp}" # where the action is
12+
master="${master:-/usr/local/www/nginx/aroundblocks.info}" # master www site
13+
urls=`cat $store/urls.txt` # this should be a text file in $store
14+
sites="echo $urls | sed 's~http[s]*://~~g'" # $urls without http/s
15+
iframe_output="{iframe_output:-$master/mirror_status.html}"
16+
tor="${tor:-/usr/local/bin/usewithtor}" # torsocks location
17+
wget="${wget:-/usr/local/bin/wget --no-check-certificate -m -P $store $urls}"
18+
now="`date "+%Y%m%d-%H:%M:%S`"
19+
20+
# check if $master is at least populated, if not a good copy
21+
22+
master_check () {
23+
24+
if [ ! -f $master/* ]; then
25+
# echo "The $master directory is empty. Nothing to diff with."
26+
logger -s "The $master directory is empty. Nothing to diff with."
27+
exit 1;
28+
else
29+
logger -s "$master seems populated, at least"
30+
fi
31+
}
32+
33+
# wget each mirror site and put into /tmp/$urls
34+
35+
get_site () {
36+
37+
if [ *.onion ]; then
38+
$tor /usr/local/bin/wget --no-check-certificate -m -P $store $urls
39+
else
40+
/usr/local/bin/wget --no-check-certificate -m -P $store $urls
41+
fi
42+
}
43+
44+
# diff the $master and mirror and output diff to /tmp
45+
46+
diff_site () {
47+
/usr/bin/diff -qr $master $store/$sites >$store/diff_$site
48+
}
49+
50+
master_check; get_site; diff_site
51+
52+
# if there's content to diff_$site add to bad_sites. If not, ignore
53+
54+
if [ ! -s "diff_$site" ]; then
55+
echo " $site" >>$store/bad_sites
56+
echo "<p> $site bad `date "+%Y%m%d-%H:%M:%S"` >>$store/bad_sites.txt
57+
logger -s "diff problem with $site"
58+
cat $store/notice | mail -s "Problem with your AroundBlocks.info mirror" $admin
59+
60+
else
61+
echo "$site good as of $now" >$iframe_output
62+
logger -s "$site good as of `date`."
63+
fi
64+
65+
true

‎mirror_check0.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/sh
2+
3+
URL="https://aroundblocks.info"
4+
SITE="aroundblocks.info"
5+
BASE="/tmp/base"
6+
STORE="/tmp" # parent base directory for everything
7+
ADMIN="[email protected]"
8+
ALERT="[email protected]"
9+
OOPS="$STORE/sorry_message"
10+
11+
# check if /tmp/base is at least populated, if not a good copy
12+
13+
base_check () {
14+
if [ ! -f $BASE/* ]; then
15+
echo "The $BASE directory is empty. Nothing to diff with."
16+
logger -s "The $BASE directory is empty. Nothing to diff with."
17+
exit 1;
18+
19+
else
20+
logger -s "$BASE seems populated, at least"
21+
fi
22+
}
23+
24+
# wget the mirror site and put into /tmp/$URL
25+
26+
get_site () {
27+
/usr/local/bin/wget --no-check-certificate -m -P $STORE $URL
28+
}
29+
30+
# diff the clean copy and mirror and output diff to /tmp
31+
32+
diff_site () {
33+
/usr/bin/diff -qr $BASE $STORE/$SITE >$STORE/diff_$SITE
34+
}
35+
36+
base_check; get_site; diff_site
37+
38+
# if there's content to diff_$SITE, notify. If not, ignore
39+
40+
if [ ! -s "diff_$SITE" ]; then
41+
echo "$SITE bad as of `date`" >>/home/gman/mirror_status.html
42+
logger -s "diff problem with $SITE"
43+
cat $OOPS | mail -s "Problem with your AroundBlocks.info mirror" $ADMIN
44+
45+
else
46+
47+
echo "$SITE good as of `date`" >>/home/gman/mirror_status.html
48+
logger -s "$SITE good as of `date`."
49+
50+
fi

‎mirror_check3.sh

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/bin/sh
2+
# 20140701
3+
# determine integrity of AroundBlocks.info mirror compared to local copy
4+
# add to mirrors.html file if good, add to bad_sites if bad
5+
6+
store="${store:-/tmp}"
7+
base="$store/base"
8+
#oops="$store/sorry_message"
9+
#url="${url:-http://dhppldk4qfqssn4x.onion}"
10+
url="${url:-http://ljqw3pmt7wgt653a.onion}"
11+
#urls="urls.txt"
12+
#site="${site:-dhppldk4qfqssn4x.onion"}
13+
site="${site:-ljqw3pmt7wgt653a.onion}"
14+
#sites="sites.txt"
15+
tor="/usr/local/bin/usewithtor"
16+
admin="${admin:-george@queair.net"}
17+
alert="$alert:-george@queair.net"}
18+
19+
# check if /tmp/base is at least populated, if not a good copy
20+
21+
base_check () {
22+
23+
if [ ! -f $base/* ]; then
24+
# echo "The $base directory is empty. Nothing to diff with."
25+
logger -s "The $base directory is empty. Nothing to diff with."
26+
exit 1;
27+
else
28+
logger -s "$base seems populated, at least"
29+
fi
30+
}
31+
32+
# wget the mirror site and put into /tmp/$url
33+
34+
get_site () {
35+
36+
if [ *.onion ]; then
37+
$tor /usr/local/bin/wget --no-check-certificate -m -P $store $url
38+
39+
else
40+
41+
/usr/local/bin/wget --no-check-certificate -m -P $store $url
42+
fi
43+
}
44+
45+
# diff the clean copy and mirror and output diff to /tmp
46+
47+
diff_site () {
48+
/usr/bin/diff -qr $base $store/$site >$store/diff_$site
49+
}
50+
51+
base_check; get_site; diff_site
52+
53+
# if there's content to diff_$site add to bad_sites. If not, ignore
54+
55+
if [ ! -s "diff_$site" ]; then
56+
echo " $site" >>$store/bad_sites
57+
# echo "<p> $site bad as of `date`" >>/usr/local/www/blocks/index.html
58+
logger -s "diff problem with $site"
59+
# cat $oops $store/diff_$site >$store/notice
60+
# cat $store/notice | mail -s "Problem with your AroundBlocks.info mirror" $admin
61+
62+
else
63+
echo "$site good as of `date`" >/usr/local/www/blocks/index.html
64+
logger -s "$site good as of `date`."
65+
fi
66+
67+
true

‎mirror_pull.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/sh
2+
# mirror_pull.sh 20140721 gman
3+
# DEPS wget, fetch
4+
# mirror a remote site based on a change to the master site
5+
6+
# variables
7+
master="${master:- # master site to mirror
8+
pull="${pull:-/usr/local/bin/wget}" # method of pull wget|fetch|whatever
9+
tor="${tor:-/usr/local/bin/usewithtor}" # torsocks location
10+
store="${store:-/tmp}" # local store for remote mirror
11+
mirror="${mirror:-/usr/local/www/mirror}" # location of local mirror
12+
13+
# functions
14+
15+
is tor socks listening? if not, and mirror is .onion, use tor2web.org
16+
17+
if it is, user can decide whether to access mirror over tor
18+
19+
grab the $mirror with wget|fetch then do store the checksum and keep the remote site to speed up differential pull

‎tar_site.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/sh
2+
# tar_site.sh 20140721 gman
3+
# DEPS none
4+
# create a tarball of www site and dump to some place with SHA256 checksum into iframe
5+
6+
# variables
7+
#wwwroot="${wwwroot:-/usr/local/www/site}" # site's www directory
8+
#storeroot="${storeroot:-/usr/local/www/storage}" # location for dumping zip file
9+
storeroot="${storeroot:-/tmp}"
10+
site_tarball="${site_tarball:-site.tar.gz}" # output zip file name
11+
tar="${tar:-tar zcf $storeroot/$site_tarball $wwwroot}" # gzip command
12+
checksum="${checksum:-/bin/sha256}" # define checksum
13+
iframe_output="${iframe_output:-$wwwroot/tar_frame.html}" # iframe output html file
14+
15+
# clean-up old
16+
17+
rm $storeroot/$site_tarball $wwwroot/$iframe_output;
18+
19+
$tar $wwwroot/;
20+
21+
# output iframe html to www page
22+
23+
echo "<a href="$storeroot/$site_tarball">aroundblocks.tar.gz</a><p>" >$iframe_output;
24+
25+
# add the SHA256 checksum to the iframe
26+
27+
$checksum $storeroot/$site_tarball | cut -d " " -f 1,4 >>$iframe_output;
28+
29+
true

‎zip_site.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/sh
2+
# zip_site.sh 20140709 gman
3+
# DEPS zip
4+
# create zip file of www site and dump to some place with SHA256 checksum into iframe
5+
6+
# variables
7+
#wwwroot="${wwwroot:-/usr/local/www/site}" # site's www directory
8+
#storeroot="${storeroot:-/usr/local/www/storage}" # location for dumping zip file
9+
storeroot="${storeroot:-/tmp}"
10+
site_zip="${site_zip:-website.zip}" # output zip file name
11+
zip="${zip:-/usr/local/bin/zip -r $storeroot/$site_zip $wwwroot}" # zip command
12+
checksum="${checksum:-/bin/sha256}" # define checksum
13+
iframe_output="${iframe_output:-$wwwroot/zip_frame.html}" # iframe output html file
14+
15+
# clean-up old
16+
17+
rm $storeroot/$site_zip $wwwroot/$iframe_output;
18+
19+
$zip $wwwroot/;
20+
21+
# output iframe html to www page
22+
23+
echo "<a href="$storeroot/$site_zip">website.zip</a><p>" >$iframe_output;
24+
25+
# add the SHA256 checksum to the iframe
26+
27+
$checksum $storeroot/$site_zip | cut -d " " -f 1,4 >>$iframe_output;
28+
29+
true

0 commit comments

Comments
 (0)