Merge pull request #454 from Googulator/source-mirrors

Support multiple mirrors for each source file

download-distfiles.sh

@@ -15,8 +15,19 @@ download_source() {
     local dest_path="${distfiles}/${fname}"
     if ! [ -e "${dest_path}" ]; then
         echo "Downloading ${fname}"
-        curl --fail --location "${url}" --output "${dest_path}"
+        curl --fail --location "${url}" --output "${dest_path}" || true
     fi
+}
+
+check_source() {
+    local distfiles="${1}"
+    local url="${2}"
+    local checksum="${3}"
+    local fname="${4}"
+    # Default to basename of url if not given
+    fname="${fname:-$(basename "${url}")}"
+
+    local dest_path="${distfiles}/${fname}"
     echo "${checksum}  ${dest_path}" | sha256sum -c
 }
 
@@ -25,6 +36,7 @@ set -e
 cd "$(dirname "$(readlink -f "$0")")"
 mkdir -p distfiles
 
+# First, try to download anything missing - ignore failing mirrors
 for entry in steps/*; do
     [ -e "${entry}/sources" ] || continue
 
@@ -35,3 +47,15 @@ for entry in steps/*; do
         download_source distfiles ${line}
     done < "${entry}/sources"
 done
+
+# Then, check if everything has been obtained at least once
+for entry in steps/*; do
+    [ -e "${entry}/sources" ] || continue
+
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        check_source distfiles ${line}
+    done < "${entry}/sources"
+done

lib/generator.py

@@ -12,6 +12,7 @@
 import os
 import shutil
 import tarfile
+import traceback
 import requests
 
 # pylint: disable=too-many-instance-attributes
@@ -305,7 +306,12 @@ def download_file(url, directory, file_name, silent=False):
     def get_packages(self):
         """Prepare remaining sources"""
         for line in self.source_manifest:
-            path = self.download_file(line[2], line[1], line[3])
+            try:
+                path = self.download_file(line[2], line[1], line[3])
+            except requests.HTTPError:
+                print(traceback.format_exc())
+        for line in self.source_manifest:
+            path = os.path.join(line[1], line[3])
             self.check_file(path, line[0])
 
     @classmethod

steps/helpers.sh

@@ -257,14 +257,25 @@ build() {
     unset extract
 }
 
-interpret_source_line() {
+download_source_line() {
     url="${1}"
     checksum="${2}"
     fname="${3}"
     # Default to basename of url if not given
     fname="${fname:-$(basename "${url}")}"
     if ! [ -e "${fname}" ]; then
-        curl --fail --retry 5 --location "${url}" --output "${fname}"
+        curl --fail --retry 5 --location "${url}" --output "${fname}" || true
+    fi
+}
+
+check_source_line() {
+    url="${1}"
+    checksum="${2}"
+    fname="${3}"
+    # Default to basename of url if not given
+    fname="${fname:-$(basename "${url}")}"
+    if ! [ -e "${fname}" ]; then
+        false
     fi
     echo "${checksum}  ${fname}" > "${fname}.sum"
     sha256sum -c "${fname}.sum"
@@ -279,7 +290,13 @@ default_src_get() {
     while read line; do
         # This is intentional - we want to split out ${line} into separate arguments.
         # shellcheck disable=SC2086
-        interpret_source_line ${line}
+        download_source_line ${line}
+    done < "${base_dir}/sources"
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        check_source_line ${line}
     done < "${base_dir}/sources"
     cd -
 }