Merge pull request #381 from kabanero-io/rc5

Cleanup

Author: aadeshpa

pipelines/experimental/gitops/build-push-promote-task.yaml

@@ -1,3 +1,14 @@
+#
+# This task will build an appsody project specificed in the git-source using `appsody build` 
+# and push the generated application image to the specified image registry. The image can be optionally
+# signed before it's pushed to the registry. 
+#
+# If the gitops-map configmap is configured, it will also promote the service to the specified gitops repo.
+# If gitops is enabled, a secret called gitops-token also needs to be configured with the token for the gitops repo.
+#
+# Insecure registy access or secure connection to the image registries can be setup by configuring the 
+# OpenShift cluster resource.  For more information, refer to https://kabanero.io/guides/working-with-pipelines/#transport-layer-security-tls-verification-for-image-registry-access-in-pipelines
+#
 apiVersion: tekton.dev/v1beta1
 kind: Task
 metadata:
@@ -19,6 +30,8 @@ spec:
     type: string
   steps:
   - name: enforce-stack-policy-pre-build
+    # This step enforces the pre build stack governance policy configured in the Kabanero CR.
+    # Refer to https://kabanero.io/docs/ref/general/reference/semver-governance.html for policy details.
     securityContext:
       privileged: true
     image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
@@ -32,52 +45,71 @@ spec:
       value: git-source
     volumeMounts:
     - mountPath: /var/lib/containers
-      name: varlibcontainers
+      name: varlibcontainers     
   - name: build
+    # This steps builds the source project using appsody build.
     securityContext:
       privileged: true
     image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
+    imagePullPolicy: Always
     command: ["/bin/bash"]
     args:
       - -c
       - |
-        cd /workspace/$gitsource
-
-        #executing the insecure_registry_setup.sh script if exists, to add internal registry to insecure registry list
-        if [ -f "/scripts/insecure_registry_setup.sh" ]; then
-           echo "Running the script /scripts/insecure_registry_setup.sh ...."
-           /scripts/insecure_registry_setup.sh
-           echo "printing the content /etc/containers/registries.conf"
-           cat /etc/containers/registries.conf
+        # Configure image registry access in the container by adding it to the insecure registry list or enabling TLS verification
+        # by adding it to the trust store based on OpenShift cluster resource configuration.
+        echo "[INFO] Running the script /scripts/image_registry_access_setup.sh ...."
+        /scripts/image_registry_access_setup.sh
+        retVal=$?
+        if [ $retVal -ne 0 ]
+        then
+           echo "[ERROR] The script failed(/scripts/image_registry_access_setup.sh)" >&2
+           exit $retVal
         fi
+        echo "[INFO] Completed setup for image registry access."
 
-        #executing the ca_certs_setup.sh script if exists, to add additional trusted ca certs to /etc/docker/certs.d/<hosname>/ca.crt
-        if [ -f "/scripts/ca_certs_setup.sh" ]; then
-           echo "Running the script /scripts/ca_certs_setup.sh ...."
-           /scripts/ca_certs_setup.sh
+        # If the image registry URL of the stack image is the external route of the internal registry, change the stack regisry URL to the 
+        # internal route.  This avoids having to configure additional secrets, certificates etc.
+        OUTPUTS_STACK_IMAGE_REGISTRY_URL=$( /scripts/stack_registry_url_setup.sh )
+        retVal=$?
+        if [ $retVal -ne 0 ]
+        then
+           echo "[ERROR] The script failed(/scripts/stack_registry_url_setup.sh) Reason: $OUTPUTS_STACK_IMAGE_REGISTRY_URL" >&2
+           exit $retVal
         fi
 
+        echo "[INFO] Stack registry URL = $OUTPUTS_STACK_IMAGE_REGISTRY_URL"
+
+        # Docker does not support upper case characters in the image name.  Github does not have this restriction.
+        # So lowercase the image name if it has any upper case characters.
         OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$( /scripts/imageurl_imagename_lowercase.sh -u $(outputs.resources.docker-image.url) -n $(inputs.params.docker-imagename) -t $(inputs.params.docker-imagetag) )
         retVal=$?
         if [ $retVal -ne 0 ]
         then
-           echo "The script failed(/scripts/imageurl_imagename_lowercase.sh) Reason: $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" >&2
+           echo "[ERROR] The script failed(/scripts/imageurl_imagename_lowercase.sh) Reason: $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" >&2
            exit $retVal
         fi
 
-        echo "OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
+        echo "[INFO] Application image URL = $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
 
+        # Kickoff appsody build
         cd /workspace/$gitsource
-        appsody build -t "$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" --buildah --buildah-options "--format=docker"
-        #echo "Copying the generated app-deploy.yaml file from input to the output to pass the file to the next task when this task is used in deploy pipeline"
-        #cp app-deploy.yaml $(outputs.resources.git-source.path)
+        echo "[INFO] Running appsody build..."
+        appsody build -t "$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" --buildah --buildah-options "--format=docker" --stack-registry "$OUTPUTS_STACK_IMAGE_REGISTRY_URL"
+        if [ $? != 0 ]; then
+            echo "[ERROR] Appsody build failed.  Please review the appsody build logs above.  Pipeline run aborted."
+            exit 1
+        fi
+        echo "[INFO] Completed appsody build."
     env:
     - name: gitsource
       value: git-source
     volumeMounts:
     - mountPath: /var/lib/containers
       name: varlibcontainers
   - name: enforce-stack-policy-post-build
+    # This step enforces the post build stack governance policy configured in the Kabanero CR.
+    # Refer to https://kabanero.io/docs/ref/general/reference/semver-governance.html for policy details.
     securityContext:
       privileged: true
     image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
@@ -86,66 +118,71 @@ spec:
       - -c
       - |
         /scripts/enforce_stack_policy.sh post-build
+        
     env:
     - name: gitsource
       value: git-source
     volumeMounts:
     - mountPath: /var/lib/containers
-      name: varlibcontainers
+      name: varlibcontainers   
   - name: push
+    # Push the image built in the build step to the specified image registry.  Optionally sign the image.
     securityContext:
       privileged: true
     image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
     command: ["/bin/bash"]
     args:
       - -c
       - |
-
+        # Docker does not support upper case characters in the image name.  Github does not have this restriction.
+        # So lowercase the image name if it has any upper case characters.
         OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$( /scripts/imageurl_imagename_lowercase.sh -u $(outputs.resources.docker-image.url) -n $(inputs.params.docker-imagename) -t $(inputs.params.docker-imagetag) )
         retVal=$?
         if [ $retVal -ne 0 ]
         then
-           echo "The script failed(/scripts/imageurl_imagename_lowercase.sh) Reason: $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" >&2
+           echo "[ERROR] The script failed(/scripts/imageurl_imagename_lowercase.sh) Reason: $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" >&2
            exit $retVal
         fi
-        echo "OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
+        
+        echo "[INFO] Application image URL = $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
 
-        #executing the insecure_registry_setup.sh script if exists, to add internal registry to insecure registry list
-        if [ -f "/scripts/insecure_registry_setup.sh" ]; then
-           echo "Running the script /scripts/insecure_registry_setup.sh ...."
-           /scripts/insecure_registry_setup.sh
-        fi
-
-        #executing the ca_certs_setup.sh script if exists, to add additional trusted ca certs to /etc/docker/certs.d/<hosname>/ca.crt
-        if [ -f "/scripts/ca_certs_setup.sh" ]; then
-           echo "Running the script /scripts/ca_certs_setup.sh ...."
-           /scripts/ca_certs_setup.sh
+        # Configure image registry access in the container by adding it to the insecure registry list or enabling TLS verification
+        # by adding it to the trust store based on OpenShift cluster resource configuration.
+        echo "[INFO] Running the script /scripts/image_registry_access_setup.sh ...."
+        /scripts/image_registry_access_setup.sh
+        retVal=$?
+        if [ $retVal -ne 0 ]
+        then
+           echo "[ERROR] The script failed(/scripts/image_registry_access_setup.sh)" >&2
+           exit $retVal
         fi
 
-        #if /image-signing-config/registry does not exist, a container image signature is not generated.
+        # Check if /image-signing-config/registry is setup to enable container image signature generation.
         if [ -f "/image-signing-config/registry" ]; then
-           REPO=`cat /image-signing-config/registry`
+            REPO=`cat /image-signing-config/registry`
         fi
         if [[ -z $REPO ]] || [[ $OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE != $REPO/* ]]; then
-           echo "Signature will not be generated. The signed image repository is not set or does not match the target registry."
+           echo "[INFO] Signature will not be generated. The signed image repository is not set or does not match the target registry."
+           echo "[INFO] Pushing image to registry..."
            buildah push "$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" "docker://$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
         else
-           echo "Signature will be generated."
+           echo "[INFO] Signature will be generated."
            #importing RSA secret key, then extract an e-mail address from it.
            gpg --import /image-signing-config/secret.asc
            SIGNBY=`gpg --list-keys|sed -n -e "/.*<.*>.*/p"|sed -e "s/^.*<\(.*\)>.*$/\1/"`
+           echo "[INFO] Pushing image to registry..."
            skopeo copy --remove-signatures --sign-by $SIGNBY "containers-storage:$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE" "docker://$OUTPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
            RESULT=$?
            if [ $RESULT -ne 0 ]; then
-              echo "sign-image failed. exit code : $RESULT"
+              echo "[ERROR] sign-image failed. exit code : $RESULT"
               exit $RESULT
            fi
            if [[ -z `cat /image-signing-config/sigstore` ]]; then
-              echo "Signature is stored in the image registry"
+              echo "[INFO] Signature is stored in the image registry"
            else
               #invoking scripts for processing a generated signature.
-              echo "A signature is stored by scripts."
-              for f in /sigstore-script/*; do [ -f "$f" ] || break; echo "Processing $f"; $f;  done
+              echo "[INFO] A signature is stored by scripts."
+              for f in /sigstore-script/*; do [ -f "$f" ] || break; echo "[INFO] Processing $f"; $f;  done
            fi
         fi
     env:
@@ -159,23 +196,24 @@ spec:
     - mountPath: /etc/containers/registries.d
       name: registries-d
     - mountPath: /sigstore-script
-      name: sigstore-script
+      name: sigstore-script    
   - name: promote
+  # Promote the service to the configured gitops repository using the `services promote` CLI
     securityContext:
       privileged: true
-    image: quay.io/redhat-developer/gitops-cli:latest
+    image: quay.io/redhat-developer/gitops-cli@sha256:f5f47bb0cf1dcd081c29e7017e4511441cab3379d06ef587ba6e539f005945dd
     command: ["/bin/bash"]
     args:
       - -c
       - |
         echo " "
-        echo "Gitops repo url = $GITOPS_REPOSITORY_URL"
-        echo "Gitops repo type = $GITOPS_REPOSITORY_TYPE"
-        echo "Gitops repo commit name = $GITOPS_COMMIT_USER_NAME"
-        echo "Gitops repo commit email = $GITOPS_COMMIT_USER_EMAIL"
-        echo " "
+        echo "[INFO] Gitops repo url = $GITOPS_REPOSITORY_URL"
 
         if [[ ! -z "$GITOPS_REPOSITORY_URL" ]]; then
+          echo "[INFO] Gitops repo type = $GITOPS_REPOSITORY_TYPE"
+          echo "[INFO] Gitops repo commit name = $GITOPS_COMMIT_USER_NAME"
+          echo "[INFO] Gitops repo commit email = $GITOPS_COMMIT_USER_EMAIL"
+          echo " "
 
           if [ -z "$GITHUB_TOKEN" ]; then
             echo "[ERROR] Secret 'gitops-token' with the token to access gitops repo was not configured. Please configure secret and try again."
@@ -229,7 +267,7 @@ spec:
           cat $HOME/.gitconfig
 
           echo " "
-          echo "service name = $(inputs.params.git-project)"
+          echo "[INFO] service name = $(inputs.params.git-project)"
           if [[ -z "$(inputs.params.git-project)" ]]; then
             echo "[ERROR] Unable to retrieve service name from input params.  Unable to promote."
             exit 1
@@ -242,7 +280,7 @@ spec:
           fi
 
           gitCommit=$(git rev-parse --short HEAD)
-          services promote --from /workspace/$gitsource --to $(GITOPS_REPOSITORY_URL) --service $(inputs.params.git-project) --commit-message "Publish $(inputs.params.git-project) commit $gitCommit" --debug  --repository-type GITOPS_REPOSITORY_TYPE
+          services promote --from /workspace/$gitsource --to $(GITOPS_REPOSITORY_URL) --service $(inputs.params.git-project) --commit-message "Publish $(inputs.params.git-project) commit $gitCommit" --repository-type $(GITOPS_REPOSITORY_TYPE)
           
           if [ $? != 0 ]; then
             echo "[ERROR] Promote to gitops repo failed.  Please review logs above."

pipelines/experimental/gitops/deploy-kustomize-pl.yaml

@@ -1,3 +1,4 @@
+# This pipeline will call a task to deploy a service using the kustomization.yaml in the gitops repo.
 apiVersion: tekton.dev/v1beta1
 kind: Pipeline
 metadata:
@@ -13,4 +14,4 @@ spec:
       resources:
         inputs:
         - name: git-source
-          resource: git-source
+          resource: git-source

pipelines/experimental/gitops/deploy-kustomize-task.yaml

@@ -1,3 +1,4 @@
+# This task will deploy the service using the kustomization.yaml in the gitops repo.
 apiVersion: tekton.dev/v1beta1
 kind: Task
 metadata:
@@ -14,21 +15,27 @@ spec:
         #!/usr/bin/env sh
         
         # First check if a dryrun is successful
-        kubectl apply -k /workspace/git-source/env --dry-run=server
+        echo "[INFO] Starting dry run..."
+        kubectl apply -k /workspace/git-source/environments --dry-run=server
         
         if [ $? != 0 ]; then
           echo "[ERROR] Dry run of deployment was unsuccessful. Please review errors above for more details. Service will not be deployed."
           exit 1
         fi
 
         # If it's good then run
+        echo "[INFO] Starting deployment..."
         kubectl apply -k /workspace/git-source/environments
+        if [ $? != 0 ]; then
+          echo "[ERROR] Deployment was unsuccessful. Please review errors above for more details. Service was not deployed."
+          exit 1
+        fi
 
-        # TODO: The pipeline declares success if apply is successful.  It takes a minute or so after we apply for the app pod to come up.
-        # Maybe we should wait and try to check for status of app?.
+        # TODO: To support newer kustomize we should be doing
+        # kustomize build /workspace/git-source/environments | kubectl apply -k -
 
-        # TODO: To support newer kustomize 
-        # kustomize build /workspace/git-source/env | kubectl apply -k -
+        # TODO: The pipeline declares success if apply is successful.  It takes a minute or so after we apply for the app pod to come up.
+        # Maybe we should wait and try to check for status of app?
     env:
     - name: gitsource
       value: git-source

pipelines/experimental/gitops/image-scan-task.yaml

@@ -37,13 +37,15 @@ spec:
     - name: mount-image          
       securityContext:
         privileged: true
-      image: kabanero/kabanero-utils:0.9.0
+      image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
 # Temporarily make copy of mounted image since the mounted image will be unmounted when the container for this task ends.
 # TODO: Determine another way to persist the mounted container image across containers
       command: ['/bin/bash']
       args:
         - -c
         - |
+          # Docker does not support upper case characters in the image name.  Github does not have this restriction.
+          # So lowercase the image name if it has any upper case characters.
           INPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$( /scripts/imageurl_imagename_lowercase.sh -u $(inputs.resources.docker-image.url) -n $(inputs.params.docker-imagename) -t $(inputs.params.docker-imagetag) )
           retVal=$?
           if [ $retVal -ne 0 ]
@@ -53,16 +55,15 @@ spec:
           fi
           echo "INPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE=$INPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
  
-          #executing the insecure_registry_setup.sh script if exists, to add internal registry to insecure registry list
-          if [ -f "/scripts/insecure_registry_setup.sh" ]; then
-             echo "Running the script /scripts/insecure_registry_setup.sh ...."
-             /scripts/insecure_registry_setup.sh
-          fi
-
-          #executing the ca_certs_setup.sh script if exists, to add additional trusted ca certs to /etc/docker/certs.d/<hosname>/ca.crt
-          if [ -f "/scripts/ca_certs_setup.sh" ]; then
-             echo "Running the script /scripts/ca_certs_setup.sh ...."
-             /scripts/ca_certs_setup.sh
+          # Configure image registry access in the container by adding it to the insecure registry list or enabling TLS verification
+          # by adding it to the trust store based on OpenShift cluster resource configuration.
+          echo "[INFO] Running the script /scripts/image_registry_access_setup.sh ...."
+          /scripts/image_registry_access_setup.sh
+          retVal=$?
+          if [ $retVal -ne 0 ]
+          then
+             echo "[ERROR] The script failed(/scripts/image_registry_access_setup.sh), and the image registry access setup was not complete, aborting the pipelinerun" >&2
+             exit $retVal
           fi
 
           echo "Pulling image docker://$INPUTS_RESOURCE_DOCKER_IMAGE_URL_LOWERCASE"
@@ -90,7 +91,7 @@ spec:
     - name: scan-image
       securityContext:
         privileged: true
-      image: kabanero/scanner:1.3.1
+      image: kabanero/scanner@sha256:82b979de485fc5990d41b8b4acc2aeee2211bd26b0f03374f5657773ae4148f9
       command: ['/bin/bash']
       args:
         - -c
@@ -189,4 +190,4 @@ spec:
       hostPath:
         path: /var/lib
     - name: varlibcontainers
-      emptyDir: {}
+      emptyDir: {}

pipelines/experimental/sample-helper-files/gitops/gitops-map.yaml

@@ -3,6 +3,7 @@ apiVersion: v1
 metadata:
   name: gitops-map
 data:
-  gitops-repo-url: <gitops_repo_url> 
+  gitops-repository-url: <gitops_repo_url> 
+  gitops-repository-type: <gitops_repo_type: ghe, github, gitlab>
   gitops-commit-user-name: <user_name_to_commit_using>
   gitops-commit-user-email: <user_email_to_commit_using>

pipelines/incubator/build-deploy-task.yaml

@@ -146,9 +146,12 @@ spec:
     - mountPath: /var/lib/containers
       name: varlibcontainers
   - name: deploy-image
-    image: kabanero/kabanero-utils:0.3.0
+    image: kabanero/kabanero-utils@sha256:8020573657ed1b80e2b872a37719d6398ee78219296fe6180733f54425f7bd6a
     command: ['/bin/sh']
-    args: ['-c', 'find /workspace/$gitsource -name ${YAMLFILE} -type f|xargs kubectl apply -f']
+    args:
+      - -c
+      - |
+         kubectl apply -f  /workspace/$gitsource/${YAMLFILE}
     env:
     - name: gitsource
       value: git-source