Load revised MBS configuration properties

lib/distrobaker.py

@@ -258,10 +258,87 @@ def load_config(crepo):
                     logger.error("Configuration error: %s.profile missing.", k)
                     return None
                 if "mbs" in cnf[k]:
-                    n[k]["mbs"] = str(cnf[k]["mbs"])
+                    # MBS properties are only relevant for the destination
+                    if k == "destination":
+                        if not isinstance(cnf[k]["mbs"], dict):
+                            logger.error(
+                                "Configuration error: %s.mbs must be a mapping.",
+                                k,
+                            )
+                            return None
+                        if "auth_method" not in cnf[k]["mbs"]:
+                            logger.error(
+                                "Configuration error: %s.mbs.%s is missing.",
+                                k,
+                                "auth_method",
+                            )
+                            return None
+                        mbs_auth_method = str(cnf[k]["mbs"]["auth_method"])
+                        mbs_required_configs = ["auth_method", "api_url"]
+                        if mbs_auth_method == "oidc":
+                            # Try to import this now so the user gets immediate
+                            # feedback if it isn't installed
+                            try:
+                                import openidc_client  # noqa: F401
+                            except ImportError:
+                                logger.exception(
+                                    "python-openidc-client needs to be "
+                                    "installed for %s.mbs.%s %s",
+                                    k,
+                                    "auth_method",
+                                    mbs_auth_method,
+                                )
+                                return None
+                            mbs_required_configs += [
+                                "oidc_id_provider",
+                                "oidc_client_id",
+                                "oidc_client_secret",
+                                "oidc_scopes",
+                            ]
+                        elif mbs_auth_method == "kerberos":
+                            # Try to import this now so the user gets immediate
+                            # feedback if it isn't installed
+                            try:
+                                import requests_kerberos  # noqa: F401
+                            except ImportError:
+                                logger.exception(
+                                    "python-requests-kerberos needs to be "
+                                    "installed for %s.mbs.%s %s",
+                                    k,
+                                    "auth_method",
+                                    mbs_auth_method,
+                                )
+                                return None
+                        else:
+                            logger.error(
+                                "Configuration error: %s.mbs.%s %s is unsupported.",
+                                k,
+                                "auth_method",
+                                mbs_auth_method,
+                            )
+                            return None
+                        n[k]["mbs"] = dict()
+                        for r in mbs_required_configs:
+                            if r not in cnf[k]["mbs"]:
+                                logger.error(
+                                    "Configuration error: %s.mbs.%s is required when %s is %s.",
+                                    k,
+                                    r,
+                                    "auth_method",
+                                    mbs_auth_method,
+                                )
+                                return None
+                            n[k]["mbs"][r] = cnf[k]["mbs"][r]
+                    else:
+                        logger.warning(
+                            "Configuration warning: %s.mbs is extraneous, ignoring.",
+                            k,
+                        )
                 else:
-                    logger.error("Configuration error: %s.mbs missing.", k)
-                    return None
+                    # MBS properties required for destination
+                    if k == "destination":
+                        logger.error("Configuration error: %s.mbs missing.", k)
+                        return None
             else:
                 logger.error("Configuration error: %s missing.", k)
                 return None
@@ -277,7 +354,7 @@ def load_config(crepo):
             return None
         if "build" in cnf:
             n["build"] = dict()
-            for k in ("prefix", "target"):
+            for k in ("prefix", "target", "platform"):
                 if k in cnf["build"]:
                     n["build"][k] = str(cnf["build"][k])
                 else:
@@ -290,6 +367,13 @@ def load_config(crepo):
                     "Configuration warning: build.scratch not defined, assuming false."
                 )
                 n["build"]["scratch"] = False
+            if ":" not in n["build"]["platform"]:
+                logger.error(
+                    "Configuration error: build.%s.%s must be in name:stream format.",
+                    k,
+                    "platform",
+                )
+                return None
         else:
             logger.error("Configuration error: build missing.")
             return None

requirements.txt

@@ -2,8 +2,10 @@ fedora_messaging>=2.0.2
 gitpython>=3.1.9
 gunicorn>=20.0.4
 koji>=1.22.1
+openidc_client
 pygobject
 pyyaml>=5.3.1
 regex>=2020.10.11
+requests_kerberos
 rpkg>=1.61
 rpm-py-installer>=1.0.0

tests/data/config/distrobaker-dest-mbs-not-mapping.yaml

@@ -0,0 +1,74 @@
+configuration:
+  source:
+    scm: https://src.fedoraproject.org/
+    cache:
+      url: https://src.fedoraproject.org/repo/pkgs
+      cgi: https://src.fedoraproject.org/repo/pkgs/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: koji
+  destination:
+    scm: ssh://pkgs.example.com/
+    cache:
+      url: http://pkgs.example.com/repo
+      cgi: http://pkgs.example.com/lookaside/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: brew
+    mbs: https://mbs.example.com/
+  trigger:
+    rpms: rawhide
+    modules: rawhide-modular
+  build:
+    prefix: git://pkgs.example.com/
+    target: fluff-42.0.0-alpha-candidate
+    scratch: false
+    platform: platform:fl42
+  git:
+    author: DistroBaker
+    email: [email protected]
+    message: >
+      Merged update from upstream sources
+
+      This is an automated DistroBaker update from upstream sources.  If you do not
+      know what this is about or would like to opt out, contact the DistroBaker maintainers.
+  control:
+    strict: false
+    build: true
+    merge: true
+    exclude:
+      rpms:
+        - firefox
+        - kernel
+        - thunderbird
+      modules:
+        - testmodule2:master
+  defaults:
+    rpms:
+      source: "%(component)s.git"
+      destination: "%(component)s.git#fluff-42.0.0-alpha"
+    modules:
+      source: "%(component)s.git#%(stream)s"
+      destination: "%(component)s.git#%(stream)s-fluff-42.0.0-alpha"
+      rpms:
+        source: "%(component)s.git"
+        destination: "%(component)s.git#stream-%(name)s-%(stream)s"
+    cache:
+      source: "%(component)s"
+      destination: "%(component)s"
+components:
+  rpms:
+    gzip:
+      source: gzip.git
+      destination: gzip.git#fluff-42.0.0-alpha-experimental
+    ipa:
+      source: freeipa.git#f33
+      cache:
+        source: freeipa
+        destination: ipa
+  modules:
+    testmodule:master:
+      destination: testmodule#stream-master-fluff-42.0.0-alpha-experimental
+      rpms:
+        componentrpm:
+          source: componentsource.git#sourcebranch
+          destination: coomponentrpm.git#fluff-42.0.0-alpha-experimental
+        anotherrpm: {}

tests/data/config/distrobaker-extraneous-source-mbs.yaml

@@ -0,0 +1,84 @@
+configuration:
+  source:
+    scm: https://src.fedoraproject.org/
+    cache:
+      url: https://src.fedoraproject.org/repo/pkgs
+      cgi: https://src.fedoraproject.org/repo/pkgs/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: koji
+    mbs: https://this.needs.to.be.ignored
+  destination:
+    scm: ssh://pkgs.example.com/
+    cache:
+      url: http://pkgs.example.com/repo
+      cgi: http://pkgs.example.com/lookaside/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: brew
+    mbs:
+      api_url: https://mbs.example.com/module-build-service/1/
+      auth_method: oidc
+      oidc_id_provider: https://id.example.com/openidc/
+      oidc_client_id: mbs-authorizer
+      oidc_client_secret: notsecret
+      oidc_scopes:
+        - openid
+        - https://id.example.com/scope/groups
+        - https://mbs.example.com/oidc/submit-build
+  trigger:
+    rpms: rawhide
+    modules: rawhide-modular
+  build:
+    prefix: git://pkgs.example.com/
+    target: fluff-42.0.0-alpha-candidate
+    scratch: false
+    platform: platform:fl42
+  git:
+    author: DistroBaker
+    email: [email protected]
+    message: >
+      Merged update from upstream sources
+
+      This is an automated DistroBaker update from upstream sources.  If you do not
+      know what this is about or would like to opt out, contact the DistroBaker maintainers.
+  control:
+    strict: false
+    build: true
+    merge: true
+    exclude:
+      rpms:
+        - firefox
+        - kernel
+        - thunderbird
+      modules:
+        - testmodule2:master
+  defaults:
+    rpms:
+      source: "%(component)s.git"
+      destination: "%(component)s.git#fluff-42.0.0-alpha"
+    modules:
+      source: "%(component)s.git#%(stream)s"
+      destination: "%(component)s.git#%(stream)s-fluff-42.0.0-alpha"
+      rpms:
+        source: "%(component)s.git"
+        destination: "%(component)s.git#stream-%(name)s-%(stream)s"
+    cache:
+      source: "%(component)s"
+      destination: "%(component)s"
+components:
+  rpms:
+    gzip:
+      source: gzip.git
+      destination: gzip.git#fluff-42.0.0-alpha-experimental
+    ipa:
+      source: freeipa.git#f33
+      cache:
+        source: freeipa
+        destination: ipa
+  modules:
+    testmodule:master:
+      destination: testmodule#stream-master-fluff-42.0.0-alpha-experimental
+      rpms:
+        componentrpm:
+          source: componentsource.git#sourcebranch
+          destination: coomponentrpm.git#fluff-42.0.0-alpha-experimental
+        anotherrpm: {}

tests/data/config/distrobaker-missing-dest-mbs.yaml

@@ -0,0 +1,73 @@
+configuration:
+  source:
+    scm: https://src.fedoraproject.org/
+    cache:
+      url: https://src.fedoraproject.org/repo/pkgs
+      cgi: https://src.fedoraproject.org/repo/pkgs/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: koji
+  destination:
+    scm: ssh://pkgs.example.com/
+    cache:
+      url: http://pkgs.example.com/repo
+      cgi: http://pkgs.example.com/lookaside/upload.cgi
+      path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
+    profile: brew
+  trigger:
+    rpms: rawhide
+    modules: rawhide-modular
+  build:
+    prefix: git://pkgs.example.com/
+    target: fluff-42.0.0-alpha-candidate
+    scratch: false
+    platform: platform:fl42
+  git:
+    author: DistroBaker
+    email: [email protected]
+    message: >
+      Merged update from upstream sources
+
+      This is an automated DistroBaker update from upstream sources.  If you do not
+      know what this is about or would like to opt out, contact the DistroBaker maintainers.
+  control:
+    strict: false
+    build: true
+    merge: true
+    exclude:
+      rpms:
+        - firefox
+        - kernel
+        - thunderbird
+      modules:
+        - testmodule2:master
+  defaults:
+    rpms:
+      source: "%(component)s.git"
+      destination: "%(component)s.git#fluff-42.0.0-alpha"
+    modules:
+      source: "%(component)s.git#%(stream)s"
+      destination: "%(component)s.git#%(stream)s-fluff-42.0.0-alpha"
+      rpms:
+        source: "%(component)s.git"
+        destination: "%(component)s.git#stream-%(name)s-%(stream)s"
+    cache:
+      source: "%(component)s"
+      destination: "%(component)s"
+components:
+  rpms:
+    gzip:
+      source: gzip.git
+      destination: gzip.git#fluff-42.0.0-alpha-experimental
+    ipa:
+      source: freeipa.git#f33
+      cache:
+        source: freeipa
+        destination: ipa
+  modules:
+    testmodule:master:
+      destination: testmodule#stream-master-fluff-42.0.0-alpha-experimental
+      rpms:
+        componentrpm:
+          source: componentsource.git#sourcebranch
+          destination: coomponentrpm.git#fluff-42.0.0-alpha-experimental
+        anotherrpm: {}

tests/data/config/distrobaker-no-source-profile.yaml

@@ -5,22 +5,31 @@ configuration:
       url: https://src.fedoraproject.org/repo/pkgs
       cgi: https://src.fedoraproject.org/repo/pkgs/upload.cgi
       path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
-    mbs: https://mbs.fedoraproject.org
   destination:
     scm: ssh://pkgs.example.com/
     cache:
       url: http://pkgs.example.com/repo
       cgi: http://pkgs.example.com/lookaside/upload.cgi
       path: "%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s"
     profile: brew
-    mbs: https://mbs.example.com
+    mbs:
+      api_url: https://mbs.example.com/module-build-service/1/
+      auth_method: oidc
+      oidc_id_provider: https://id.example.com/openidc/
+      oidc_client_id: mbs-authorizer
+      oidc_client_secret: notsecret
+      oidc_scopes:
+        - openid
+        - https://id.example.com/scope/groups
+        - https://mbs.example.com/oidc/submit-build
   trigger:
     rpms: rawhide
     modules: rawhide-modular
   build:
     prefix: git://pkgs.example.com/
     target: fluff-42.0.0-alpha-candidate
     scratch: false
+    platform: platform:fl42
   git:
     author: DistroBaker
     email: [email protected]