Skip to content

Commit 60de1d8

Browse files
deajandeajan
committed
Update restic binaries on tests/compilation
1 parent 585f8a6 commit 60de1d8

File tree

4 files changed

+121
-51
lines changed

4 files changed

+121
-51
lines changed

‎RESTIC_SOURCE_FILES/update_restic.py

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#! /usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
4+
5+
__intname__ = "npbackup.restic_update"
6+
__author__ = "Orsiris de Jong"
7+
__copyright__ = "Copyright (C) 2024 NetInvent"
8+
__license__ = "BSD-3-Clause"
9+
__build__ = "2024121001"
10+
11+
import os
12+
import sys
13+
import bz2
14+
from pathlib import Path
15+
import requests
16+
import json
17+
import shutil
18+
from pprint import pprint
19+
20+
21+
sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
22+
23+
from npbackup.path_helper import BASEDIR
24+
25+
26+
def download_restic_binaries(arch: str = "amd64"):
27+
"""
28+
We must first download latest restic binaries to make sure we can run all tests and/or compile
29+
"""
30+
org = "restic"
31+
repo = "restic"
32+
33+
response = requests.get(
34+
f"https://api.github.com/repos/{org}/{repo}/releases/latest"
35+
)
36+
# print("RESPONSE: ", response)
37+
json_response = json.loads(response.text)
38+
current_version = json_response["tag_name"].lstrip("v")
39+
# print("JSON RESPONSE")
40+
# pprint(json_response, indent=5)
41+
42+
dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES")
43+
if os.name == "nt":
44+
fname = f"_windows_{arch}.zip"
45+
suffix = ".exe"
46+
else:
47+
fname = f"_linux_{arch}.bz2"
48+
suffix = ""
49+
50+
dest_file = dest_dir.joinpath("restic_" + current_version + fname).with_suffix(
51+
suffix
52+
)
53+
if dest_file.is_file():
54+
print(f"RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING {dest_file}")
55+
return True
56+
else:
57+
print(f"DOWNALOADING RESTIC {dest_file}")
58+
59+
downloaded = False
60+
for entry in json_response["assets"]:
61+
if fname in entry["browser_download_url"]:
62+
file_request = requests.get(
63+
entry["browser_download_url"], allow_redirects=True
64+
)
65+
print("FILE REQUEST RESPONSE", file_request)
66+
filename = entry["browser_download_url"].rsplit("/", 1)[1]
67+
full_path = dest_dir.joinpath(filename)
68+
print("PATH TO DOWNLOADED ARCHIVE: ", full_path)
69+
if fname.endswith("bz2"):
70+
with open(full_path.with_suffix(""), "wb") as fp:
71+
fp.write(bz2.decompress(file_request.content))
72+
# We also need to make that file executable
73+
os.chmod(full_path.with_suffix(""), 0o775)
74+
else:
75+
with open(full_path, "wb") as fp:
76+
fp.write(file_request.content)
77+
# Assume we have a zip or tar.gz
78+
shutil.unpack_archive(full_path, dest_dir)
79+
try:
80+
shutil.move(full_path, dest_dir.joinpath("ARCHIVES").joinpath(filename))
81+
except OSError:
82+
print(
83+
f'CANNOT MOVE TO ARCHIVE: {full_path} to {dest_dir.joinpath("ARCHIVES").joinpath(filename)}'
84+
)
85+
return False
86+
print(f"DOWNLOADED {dest_dir}")
87+
downloaded = True
88+
break
89+
if not downloaded:
90+
print(f"NO RESTIC BINARY FOUND for {arch}")
91+
return False
92+
return True
93+
94+
95+
def download_restic_binaries_for_arch():
96+
"""
97+
Shortcut to be used in compile script
98+
"""
99+
if os.name == "nt":
100+
if not download_restic_binaries("amd64") or not download_restic_binaries("386"):
101+
sys.exit(1)
102+
else:
103+
if (
104+
not download_restic_binaries("amd64")
105+
or not download_restic_binaries("arm64")
106+
or not download_restic_binaries("arm")
107+
):
108+
sys.exit(1)
109+
110+
111+
if __name__ == "__main__":
112+
download_restic_binaries_for_arch()

‎bin/COMPILE.cmd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ git pull || exit 1
1717
SET OLD_PYTHONPATH=%PYTHONPATH%
1818
SET PYTHONPATH=c:\GIT\npbackup
1919

20+
%PYTHON64% RESTIC_SOURCE_FILES/update_restic.py || exit 1
21+
2022
"%PYTHON64%" -m pip install --upgrade -r npbackup/requirements.txt || exit 1
2123
"%PYTHON64%" bin\compile.py --audience all --sign "C:\ODJ\KEYS\NetInventEV.dat"
2224

‎bin/COMPILE.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
# This is an example compiler script
44

5-
machine="$(uname -m)"
6-
75
cd /opt/npbackup
86
git pull || exit 1
97

@@ -13,7 +11,10 @@ export PYTHONPATH=/opt/npbackup
1311
# For RHEL 7 based builds, we need to define path to locally built tcl8.6
1412
[ -d /usr/local/lib/tcl8.6 ] && export LD_LIBRARY_PATH=/usr/local/lib
1513

14+
/opt/npbackup/venv/bin/python RESTIC_SOURCE_FILES/update_restic.py || exit 1
15+
16+
1617
/opt/npbackup/venv/bin/python -m pip install --upgrade -r npbackup/requirements.txt || exit 1
17-
/opt/npbackup/venv/bin/python bin/compile.py --audience all $opts
18+
/opt/npbackup/venv/bin/python bin/compile.py --audience all $@
1819

1920
export PYTHONPATH="$OLD_PYTHONPATH"

‎tests/test_npbackup-cli.py

Lines changed: 3 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@
2727
import requests
2828
import tempfile
2929
import bz2
30-
import fileinput
3130
from pprint import pprint
3231

3332
sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
3433

3534
from npbackup import __main__
3635
from npbackup.path_helper import BASEDIR
3736
from npbackup.configuration import load_config, get_repo_config
37+
from RESTIC_SOURCE_FILES.update_restic import download_restic_binaries
3838

3939

4040
if os.name == "nt":
@@ -89,54 +89,9 @@ def __str__(self):
8989
def test_download_restic_binaries():
9090
"""
9191
We must first download latest restic binaries to make sure we can run all tests
92+
Currently we only run these on amd64
9293
"""
93-
org = "restic"
94-
repo = "restic"
95-
96-
dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES")
97-
response = requests.get(
98-
f"https://api.github.com/repos/{org}/{repo}/releases/latest"
99-
)
100-
print("RESPONSE: ", response)
101-
json_response = json.loads(response.text)
102-
103-
if os.name == "nt":
104-
fname = "_windows_amd64.zip"
105-
suffix = ".exe"
106-
else:
107-
fname = "_linux_amd64.bz2"
108-
suffix = ""
109-
110-
if dest_dir.joinpath(fname).with_suffix(suffix).is_file():
111-
print("RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING")
112-
return
113-
114-
print("JSON RESPONSE")
115-
pprint(json_response, indent=5)
116-
117-
for entry in json_response["assets"]:
118-
if fname in entry["browser_download_url"]:
119-
file_request = requests.get(
120-
entry["browser_download_url"], allow_redirects=True
121-
)
122-
print("FILE REQUEST RESPONSE", file_request)
123-
filename = entry["browser_download_url"].rsplit("/", 1)[1]
124-
full_path = dest_dir.joinpath(filename)
125-
print("PATH TO DOWNLOADED ARCHIVE: ", full_path)
126-
if fname.endswith("bz2"):
127-
with open(full_path.with_suffix(""), "wb") as fp:
128-
fp.write(bz2.decompress(file_request.content))
129-
# We also need to make that file executable
130-
os.chmod(full_path.with_suffix(""), 0o775)
131-
else:
132-
with open(full_path, "wb") as fp:
133-
fp.write(file_request.content)
134-
# Assume we have a zip or tar.gz
135-
shutil.unpack_archive(full_path, dest_dir)
136-
try:
137-
shutil.move(full_path, dest_dir.joinpath("ARCHIVES"))
138-
except OSError:
139-
pass
94+
assert download_restic_binaries("amd64"), "Could not download restic binaries"
14095

14196

14297
def test_npbackup_cli_no_config():

0 commit comments

Comments
 (0)