WIP fixes to use actual un-mangled name:stream

lib/distrobaker.py

@@ -707,7 +707,7 @@ def sync_module_components(comp, nvr, modulemd=None):
 
     :param comp: The modular component name
     :param nvr: NVR of module to synchronize
-    :param modulemd: modulemd of module from build system
+    :param modulemd: Optional modulemd for module from build system
     :returns: True if successful, or False on error
     """
     logger.debug("Synchronizing components for module %s: %s", comp, nvr)
@@ -1274,6 +1274,28 @@ def process_components(compset):
     if "main" not in c:
         logger.critical("DistroBaker is not configured, aborting.")
         return None
+
+    # Generate a dictionary (key module:stream, value nvr) of all of the latest
+    # modular builds for the source tag.
+    # Note: Querying tagged modules with latest=True only returns the latest
+    # module tagged by name without regard to stream. So, we need need to query
+    # everything to figure out the latest per stream ourselves. It helps that
+    # the list returned by Koji is ordered so the most recently tagged builds
+    # are at the end of the list. This also fetches and uses the actual
+    # un-mangled name and stream for each module.
+    latest = dict()
+    for x in get_buildsys("source").listTagged(
+        c["main"]["trigger"]["modules"],
+    ):
+        binfo = get_build_info(x["nvr"])
+        if binfo is None or binfo["name"] is None or binfo["stream"] is None:
+            logger.error(
+                "Could not get module info for %s, skipping.",
+                x["nvr"],
+            )
+        else:
+            latest["{}:{}".format(binfo["name"], binfo["stream"])] = x["nvr"]
+
     if not compset:
         logger.debug(
             "No components selected, gathering components from triggers."
@@ -1284,16 +1306,9 @@ def process_components(compset):
                 c["main"]["trigger"]["rpms"], latest=True
             )
         )
-        # Note: querying tagged modules with latest=True only returns the
-        # latest module tagged for any stream, so we need need to query all
-        # and let set uniqueness eliminate duplicates
-        compset.update(
-            "{}/{}:{}".format("modules", x["package_name"], x["version"])
-            for x in get_buildsys("source").listTagged(
-                c["main"]["trigger"]["modules"],
-            )
-        )
+        compset.update("{}/{}".format("modules", x) for x in latest.keys())
     logger.info("Processing %d component(s).", len(compset))
+
     processed = 0
     for rec in sorted(compset, key=str.lower):
         m = cre.match(rec)
@@ -1319,7 +1334,11 @@ def process_components(compset):
                 m["component"],
             )
             continue
-        ref = sync_repo(comp=m["component"], ns=m["namespace"])
+        ref = sync_repo(
+            comp=m["component"],
+            ns=m["namespace"],
+            nvr=latest.get(m["component"]),
+        )
         if ref is not None:
             build_comp(comp=m["component"], ref=ref, ns=m["namespace"])
         logger.info("Done processing %s.", rec)
@@ -1333,8 +1352,8 @@ def process_components(compset):
 
 
 def get_build_info(nvr):
-    """Get SCMURL plus extra modular attributes for a source build system build
-    NVR.  NVRs are unique.
+    """Get SCMURL, plus extra attributes for modules, for a source build system
+    build NVR.  NVRs are unique.
 
     :param nvr: The build NVR to look up
     :returns: A dictionary with `scmurl`, `name`, `stream`, and `modulemd` keys,
@@ -1357,37 +1376,31 @@ def get_build_info(nvr):
             "An error occured while retrieving the build info for %s.", nvr
         )
         return None
+
+    bi = dict()
     if "source" in bsrc:
-        scmurl = bsrc["source"]
-        logger.debug("Retrieved SCMURL for %s: %s", nvr, scmurl)
+        bi["scmurl"] = bsrc["source"]
+        logger.debug("Retrieved SCMURL for %s: %s", nvr, bi["scmurl"])
     else:
-        logger.error("Cannot find any SCMURLs associated with %s.", nvr)
+        logger.error("Cannot find any SCMURL associated with %s.", nvr)
         return None
 
     try:
         minfo = bsrc["extra"]["typeinfo"]["module"]
-        mmdstr = minfo["modulemd_str"]
-        mname = minfo["name"]
-        sname = minfo["stream"]
-    except Exception:
-        mmdstr = None
-        mname = None
-        sname = None
-
-    if mmdstr:
+        bi["name"] = minfo["name"]
+        bi["stream"] = minfo["stream"]
+        bi["modulemd"] = minfo["modulemd_str"]
         logger.debug("Module info for %s:", nvr)
-        logger.debug("  name: %s", mname)
-        logger.debug("  stream: %s", sname)
-        logger.debug("  modulemd: %s", mmdstr)
-    else:
+        logger.debug("  name: %s", bi["name"])
+        logger.debug("  stream: %s", bi["stream"])
+        logger.debug("  modulemd: %s", bi["modulemd"])
+    except Exception:
+        bi["name"] = None
+        bi["stream"] = None
+        bi["modulemd"] = None
         logger.debug("No module info for %s.", nvr)
 
-    return {
-        "scmurl": scmurl,
-        "name": mname,
-        "stream": sname,
-        "modulemd": mmdstr,
-    }
+    return bi
 
 
 def get_build(comp, ns="rpms"):
@@ -1439,12 +1452,8 @@ def get_build(comp, ns="rpms"):
         cname = ms["name"]
         sname = ms["stream"]
         try:
-            # Note: querying with latest=True only returns the latest module
-            # tagged irregardless of the stream, so we need need to query all
-            # and sort out the latest per stream ourselves
             builds = bsys.listTagged(
                 c["main"]["trigger"][ns],
-                package=cname,
             )
         except Exception:
             logger.exception(
@@ -1462,22 +1471,26 @@ def get_build(comp, ns="rpms"):
             ns,
             cname,
         )
-        # try to find the latest build for the specified stream
+        # find the latest build for name:stream
         latest = None
         for b in builds:
-            if b["version"] != sname:
-                continue
-            if latest and b["creation_time"] <= latest["creation_time"]:
-                continue
-            latest = b
+            binfo = get_build_info(b["nvr"])
+            if (
+                binfo is None
+                or binfo["name"] is None
+                or binfo["stream"] is None
+            ):
+                logger.error(
+                    "Could not get module info for %s, skipping.",
+                    b["nvr"],
+                )
+            elif cname == binfo["name"] and sname == binfo["stream"]:
+                latest = b["nvr"]
         if latest:
             logger.debug(
-                "Located the latest build for %s/%s: %s",
-                ns,
-                comp,
-                latest["nvr"],
+                "Located the latest build for %s/%s: %s", ns, comp, latest
             )
-            return latest["nvr"]
+            return latest
         logger.error("Did not find any builds for %s/%s.", ns, comp)
         return None