Merge pull request #176 from fflorent/fix-wrong-asset-404-error

Extract base tag href for resolving asset URLs

lib/curl_tests.py

@@ -6,7 +6,7 @@
 import tempfile
 import pycurl
 from bs4 import BeautifulSoup
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urljoin
 from io import BytesIO
 
 DOMAIN = os.environ["DOMAIN"]
@@ -67,7 +67,8 @@ def curl(
     domain = base_url.replace("https://", "").replace("http://", "").split("/")[0]
 
     c = pycurl.Curl()  # curl
-    c.setopt(c.URL, f"{base_url}{path}")  # https://domain.tld/foo/bar
+    resolved_url = urljoin(base_url, path)
+    c.setopt(c.URL, resolved_url)  # https://domain.tld/foo/bar
     c.setopt(c.FOLLOWLOCATION, True)  # --location
     c.setopt(c.SSL_VERIFYPEER, False)  # --insecure
     c.setopt(
@@ -102,7 +103,7 @@ def curl(
 
     c.close()
 
-    return (return_code, return_content, effective_url)
+    return (return_code, return_content, effective_url, resolved_url)
 
 
 def test(
@@ -121,7 +122,7 @@ def test(
         cookies = tempfile.NamedTemporaryFile().name
 
         if DIST == "bullseye":
-            code, content, log_url = curl(
+            code, content, log_url, _ = curl(
                 f"https://{DOMAIN}/yunohost/sso",
                 "/",
                 save_cookies=cookies,
@@ -132,7 +133,7 @@ def test(
                 code == 200 and os.system(f"grep -q '{DOMAIN}' {cookies}") == 0
             ), f"Failed to log in: got code {code} or cookie file was empty?"
         else:
-            code, content, _ = curl(
+            code, content, _, _ = curl(
                 f"https://{domain}/yunohost/portalapi",
                 "/login",
                 save_cookies=cookies,
@@ -148,7 +149,7 @@ def test(
     retried = 0
     while code is None or code in {502, 503, 504}:
         time.sleep(retried * 5)
-        code, content, effective_url = curl(
+        code, content, effective_url, _ = curl(
             base_url, path, post=post, use_cookies=cookies
         )
         retried += 1
@@ -167,6 +168,9 @@ def test(
     content = content.get_text().strip() if content else ""
     content = re.sub(r"[\t\n\s]{3,}", "\n\n", content)
 
+    base_tag = html.find('base')
+    base = base_tag['href'] if base_tag else ''
+
     errors = []
     if expect_effective_url is None and "/yunohost/sso" in effective_url:
         errors.append(
@@ -232,18 +236,18 @@ def test(
             elif asset.startswith(f"{domain}/"):
                 asset = asset.replace(f"{domain}/", "")
             if not asset.startswith("/"):
-                asset = "/" + asset
-            asset_code, _, effective_asset_url = curl(
+                asset = urljoin(base + "/", asset)
+            asset_code, _, effective_asset_url, resolved_url = curl(
                 f"https://{domain}", asset, use_cookies=cookies
             )
             if asset_code != 200:
                 errors.append(
-                    f"Asset https://{domain}{asset} (automatically derived from the page's html) answered with code {asset_code}, expected 200? Effective url: {effective_asset_url}"
+                    f"Asset https://{resolved_url} (automatically derived from the page's html) answered with code {asset_code}, expected 200? Effective url: {effective_asset_url}"
                 )
-            assets.append((domain + asset, asset_code))
+            assets.append((resolved_url, asset_code))
 
     return {
-        "url": f"{base_url}{path}",
+        "url": f"{urljoin(base_url, path)}",
         "effective_url": effective_url,
         "code": code,
         "title": title,