RHTAP-3056 GitLab CI tests fixes and refactoring.

jkopriva · jkopriva · commit 26e09aabe366 · 2024-10-22T09:50:08.000+02:00
Signed-off-by: Josef Kopriva &lt;jkopriva@redhat.com&gt;
diff --git a/default.env b/default.env
@@ -67,4 +67,19 @@ export QUAY_USERNAME=''
 # Quay Password - needed for GitLab CI, GitHub Actions
 # Required: no (by default testing on Jenkins is disabled)
 # Example: 112b49e9....
-export QUAY_PASSWORD=""
+export QUAY_PASSWORD=''
+
+# Cosign public key in base64 format - needed for GitLab CI, GitHub Actions, Jenkins
+# Required: no (by default testing on Jenkins is disabled)
+# Example: 112b49e9....
+export COSIGN_PUBLIC_KEY=''
+
+# Cosign private key in base64 format - needed for GitLab CI, GitHub Actions, Jenkins
+# Required: no (by default testing on Jenkins is disabled)
+# Example: 112b49e9....
+export COSIGN_SECRET_KEY=''
+
+# Cosign secret password in base64 format - needed for GitLab CI, GitHub Actions, Jenkins
+# Required: no (by default testing on Jenkins is disabled)
+# Example: 112b49e9....
+export COSIGN_SECRET_PASSWORD=''
diff --git a/src/apis/ci/jenkins.ts b/src/apis/ci/jenkins.ts
@@ -92,8 +92,13 @@ export class JenkinsCI extends Utils {
             }
             return response.status === 200;
         } catch (error) {
-            if (error.response && error.response.status === 404) {
-                return false;
+            if (axios.isAxiosError(error)) {
+                if (error.response && error.response.status === 404) {
+                    return false;
+                } else {
+                    console.error('Axios error checking job existence:', error);
+                    throw error;
+                }
             } else {
                 console.error('Error checking job existence:', error);
                 throw error;
@@ -206,7 +211,4 @@ export class JenkinsCI extends Utils {
             console.error('Error deleting job:', error);
         }
     }
-
-
-
 }
diff --git a/src/apis/git-providers/gitlab.ts b/src/apis/git-providers/gitlab.ts
@@ -32,7 +32,7 @@ export class GitLabProvider extends Utils {
             try {
                 const projects = await this.gitlab.Projects.show(`${namespace}/${repoName}`);
                 if (projects) {
-                    console.info(`Repository with name '${repoName}' found in namespace '${namespace}'
+                    console.info(`Repository with name '${repoName}' found in organization '${namespace}'
                         created at '${projects.created_at}' url: gitlab.com/${namespace}/${repoName}`);
                     return projects.id
                 }
@@ -61,7 +61,6 @@ export class GitLabProvider extends Utils {
                 await this.sleep(5000); // Wait 5 seconds before checking again
             } catch (error) {
                 console.error('Error checking for folder creation:', error);
-                throw error;
             }
         }
     }
@@ -271,25 +270,21 @@ export class GitLabProvider extends Utils {
         }
     }
 
-        public async getLatestPipeline(projectId: number) {
-            try {
-                // const response = await this.gitlab.Pipelines.show(projectId, 'latest');
-                // console.log('Pipeline triggered successfully:', response);
-                // return response;
-
-                const pipelines = await this.gitlab.Pipelines.all(projectId);
-                if (pipelines.length === 0) {
-                  console.log(`No pipelines found!`);
-                  return null;
-                }
-                const latestPipeline = pipelines.sort((a, b) => b.id - a.id)[0];
-                console.log(`Latest pipeline ID: ${latestPipeline.id} Status: ${latestPipeline.status}`);
-                return latestPipeline;
-            } catch (error) {
-                console.error('Error triggering pipeline:', error);
-                throw error;
+    public async getLatestPipeline(projectId: number) {
+        try {
+            const pipelines = await this.gitlab.Pipelines.all(projectId);
+            if (pipelines.length === 0) {
+                console.log(`No pipelines found!`);
+                return null;
             }
+            const latestPipeline = pipelines.sort((a, b) => b.id - a.id)[0];
+            console.log(`Latest pipeline ID: ${latestPipeline.id} Status: ${latestPipeline.status}`);
+            return latestPipeline;
+        } catch (error) {
+            console.error('Error triggering pipeline:', error);
+            throw error;
         }
+    }
 
     // Trigger a GitLab pipeline
     public async triggerPipeline(projectId: number, branchName: string, triggerToken: string) {
@@ -321,7 +316,6 @@ export class GitLabProvider extends Utils {
                 await this.sleep(5000); // Wait 5 seconds before checking again
             } catch (error) {
                 console.error('Error checking for pipeline creation:', error);
-                throw error;
             }
         }
     }
@@ -346,7 +340,6 @@ export class GitLabProvider extends Utils {
                 await this.sleep(5000); // Wait 5 seconds
             } catch (error) {
                 console.error('Error checking pipeline status:', error);
-                throw error;
             }
         }
     }
@@ -399,7 +392,7 @@ export class GitLabProvider extends Utils {
             const fileContent = Buffer.from(file.content, 'base64').toString('utf-8');
 
             // Replace specific text
-            const updatedContent = fileContent.replace(textToReplace,replacement);
+            const updatedContent = fileContent.replace(textToReplace, replacement);
 
             // Encode the updated content to base64
             const encodedContent = Buffer.from(updatedContent).toString('base64');
diff --git a/src/apis/kubernetes/kube.ts b/src/apis/kubernetes/kube.ts
@@ -334,7 +334,7 @@ export class Kubernetes extends Utils {
             const secret = await k8sApi.readNamespacedSecret(secretName, namespace);
 
             // Check if the key exists in the secret data
-            if (secret.body.data && secret.body.data[keyName]) {
+            if (secret.body.data && secret.body?.data[keyName]) {
                 // Decode the base64 encoded secret value
                 const secretValue = Buffer.from(secret.body.data[keyName], 'base64').toString('utf-8');
                 return secretValue;
@@ -428,7 +428,7 @@ export class Kubernetes extends Utils {
 * @returns {Promise<string>}  - returns route URL.
 */
     public async getCosignPublicKey(): Promise<string> {
-        return this.getSecretPartialName("rhtap-app-development", "cosign-pub", "cosign.pub", true);
+        return this.getSecretPartialName("rhtap-app-development", "cosign-pub", "cosign.pub", false);
     }
 
     /**
@@ -438,7 +438,7 @@ export class Kubernetes extends Utils {
 * @returns {Promise<string>}  - returns route URL.
 */
     public async getCosignPrivateKey(): Promise<string> {
-        return this.getSecretPartialName("rhtap-tas", "fulcio-cert-trusted-artifact-signer", "private", true);
+        return this.getSecretPartialName("rhtap-tas", "fulcio-cert-trusted-artifact-signer", "private", false);
     }
 
     /**
diff --git a/src/utils/test.utils.ts b/src/utils/test.utils.ts
@@ -3,6 +3,8 @@ import { GitHubProvider } from "../../src/apis/git-providers/github";
 import { Kubernetes } from "../../src/apis/kubernetes/kube";
 import { DeveloperHubClient } from "../../src/apis/backstage/developer-hub";
 import { JenkinsCI } from "../../src/apis/ci/jenkins";
+import { syncArgoApplication } from "./argocd";
+import { TaskIdReponse } from "../../src/apis/backstage/types";
 
 
 export async function cleanAfterTestGitHub(gitHubClient: GitHubProvider, kubeClient: Kubernetes, rootNamespace: string, githubOrganization: string, repositoryName: string) {
@@ -96,4 +98,57 @@ export async function getGitLabProvider(kubeClient: Kubernetes) {
     }
 }
 
+export async function getCosignPassword(kubeClient: Kubernetes) {
+    if (process.env.COSIGN_SECRET_PASSWORD) {
+        return process.env.COSIGN_SECRET_PASSWORD;
+    } else {
+        return await kubeClient.getCosignPassword();
+    }
+}
+
+export async function getCosignPrivateKey(kubeClient: Kubernetes) {
+    if (process.env.COSIGN_SECRET_KEY) {
+        return process.env.COSIGN_SECRET_KEY;
+    } else {
+        return await kubeClient.getCosignPrivateKey();
+    }
+}
+
+export async function getCosignPublicKey(kubeClient: Kubernetes) {
+    if (process.env.COSIGN_PUBLIC_KEY) {
+        return process.env.COSIGN_PUBLIC_KEY;
+    } else {
+        return await kubeClient.getCosignPublicKey();
+    }
+}
+
+export async function waitForComponentCreation(backstageClient: DeveloperHubClient, repositoryName: string, developerHubTask: TaskIdReponse) {
+    const taskCreated = await backstageClient.getTaskProcessed(developerHubTask.id, 120000)
+
+    if (taskCreated.status !== 'completed') {
+        console.log("Failed to create backstage task. Creating logs...");
+
+        try {
+            const logs = await backstageClient.getEventStreamLog(taskCreated.id)
+            await backstageClient.writeLogsToArtifactDir('backstage-tasks-logs', `gitlab-${repositoryName}.log`, logs)
+        } catch (error) {
+            throw new Error(`Failed to write logs to artifact directory: ${error}`);
+        }
+    } else {
+        console.log("Task created successfully in backstage");
+    }
+}
+
+export async function waitForArgoSyncAndRouteContent(kubeClient: Kubernetes, backstageClient: DeveloperHubClient, developmentNamespace: string, developmentEnvironmentName: string, repositoryName: string, stringOnRoute: string) {
+    console.log("syncing argocd application in development environment")
+    await syncArgoApplication(await getRHTAPRootNamespace(), `${repositoryName}-${developmentEnvironmentName}`)
+    const componentRoute = await kubeClient.getOpenshiftRoute(repositoryName, developmentNamespace)
+    const isReady = await backstageClient.waitUntilComponentEndpointBecomeReady(`https://${componentRoute}`, 10 * 60 * 1000)
+    if (!isReady) {
+        throw new Error("Component seems was not synced by ArgoCD in 10 minutes");
+    }
+    expect(await waitForStringInPageContent(`https://${componentRoute}`, stringOnRoute, 120000)).toBe(true)
+
+}
+
 
diff --git a/tests/gpts/gitlab/suites-config/gitlab_gitlabci_suite.ts b/tests/gpts/gitlab/suites-config/gitlab_gitlabci_suite.ts
@@ -5,7 +5,7 @@ import { GitLabProvider } from "../../../../src/apis/git-providers/gitlab";
 import { Kubernetes } from "../../../../src/apis/kubernetes/kube";
 import { ScaffolderScaffoldOptions } from "@backstage/plugin-scaffolder-react";
 import { generateRandomChars } from "../../../../src/utils/generator";
-import { cleanAfterTestGitLab, getDeveloperHubClient, getGitLabProvider, getRHTAPRootNamespace, waitForStringInPageContent } from "../../../../src/utils/test.utils";
+import { cleanAfterTestGitLab, getCosignPassword, getCosignPrivateKey, getCosignPublicKey, getDeveloperHubClient, getGitLabProvider, getRHTAPRootNamespace, waitForArgoSyncAndRouteContent, waitForComponentCreation, waitForStringInPageContent } from "../../../../src/utils/test.utils";
 import { syncArgoApplication } from "../../../../src/utils/argocd";
 
 /**
@@ -113,20 +113,7 @@ export const gitLabProviderGitLabCITests = (softwareTemplateName: string, string
             * If the task is not completed within the timeout, it writes logs to the specified directory.
         */
         it(`waits for ${softwareTemplateName} component creation to finish`, async () => {
-            const taskCreated = await backstageClient.getTaskProcessed(developerHubTask.id, 120000)
-
-            if (taskCreated.status !== 'completed') {
-                console.log("Failed to create backstage task. Creating logs...");
-
-                try {
-                    const logs = await backstageClient.getEventStreamLog(taskCreated.id)
-                    await backstageClient.writeLogsToArtifactDir('backstage-tasks-logs', `gitlab-${repositoryName}.log`, logs)
-                } catch (error) {
-                    throw new Error(`Failed to write logs to artifact directory: ${error}`);
-                }
-            } else {
-                console.log("Task created successfully in backstage");
-            }
+            await waitForComponentCreation(backstageClient, repositoryName, developerHubTask);
         }, 120000);
 
         /**
@@ -162,9 +149,9 @@ export const gitLabProviderGitLabCITests = (softwareTemplateName: string, string
     * Waits for the specified ArgoCD application associated with the DeveloperHub task to be synchronized in the cluster.
 */
         it(`Setup creds for ${softwareTemplateName} pipeline`, async () => {
-            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_PUBLIC_KEY", await kubeClient.getCosignPublicKey());
-            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_SECRET_KEY", await kubeClient.getCosignPrivateKey());
-            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_SECRET_PASSWORD", await kubeClient.getCosignPassword());
+            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_PUBLIC_KEY", await getCosignPublicKey(kubeClient));
+            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_SECRET_KEY", await getCosignPrivateKey(kubeClient));
+            await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "COSIGN_SECRET_PASSWORD", await getCosignPassword(kubeClient));
             await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "GITOPS_AUTH_USERNAME", 'fakeUsername');
             await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "GITOPS_AUTH_PASSWORD", await gitLabProvider.getGitlabToken());
             await gitLabProvider.setEnvironmentVariable(gitlabRepositoryID, "QUAY_IO_CREDS_PSW", process.env.QUAY_PASSWORD || '');
@@ -199,14 +186,7 @@ export const gitLabProviderGitLabCITests = (softwareTemplateName: string, string
          * Obtain the openshift Route for the component and verify that the previous builded image was synced in the cluster and deployed in development environment
          */
         it('container component is successfully synced by gitops in development environment', async () => {
-            console.log("syncing argocd application in development environment")
-            await syncArgoApplication(RHTAPRootNamespace, `${repositoryName}-${developmentEnvironmentName}`)
-            const componentRoute = await kubeClient.getOpenshiftRoute(repositoryName, developmentNamespace)
-            const isReady = await backstageClient.waitUntilComponentEndpointBecomeReady(`https://${componentRoute}`, 10 * 60 * 1000)
-            if (!isReady) {
-                throw new Error("Component seems was not synced by ArgoCD in 10 minutes");
-            }
-            expect(await waitForStringInPageContent(`https://${componentRoute}`, stringOnRoute, 120000)).toBe(true)
+            await waitForArgoSyncAndRouteContent(kubeClient, backstageClient, developmentNamespace, developmentEnvironmentName, repositoryName, stringOnRoute);
         }, 900000)
 
         /**

Original file line number	Diff line number	Diff line change
`@@ -92,8 +92,13 @@ export class JenkinsCI extends Utils {`
`92`	`92`	`}`
`93`	`93`	`return response.status === 200;`
`94`	`94`	`} catch (error) {`
`95`		`- if (error.response && error.response.status === 404) {`
`96`		`- return false;`
	`95`	`+ if (axios.isAxiosError(error)) {`
	`96`	`+ if (error.response && error.response.status === 404) {`
	`97`	`+ return false;`
	`98`	`+ } else {`
	`99`	`+ console.error('Axios error checking job existence:', error);`
	`100`	`+ throw error;`
	`101`	`+ }`
`97`	`102`	`} else {`
`98`	`103`	`console.error('Error checking job existence:', error);`
`99`	`104`	`throw error;`
`@@ -206,7 +211,4 @@ export class JenkinsCI extends Utils {`
`206`	`211`	`console.error('Error deleting job:', error);`
`207`	`212`	`}`
`208`	`213`	`}`
`209`		`-`
`210`		`-`
`211`		`-`
`212`	`214`	`}`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ export class GitLabProvider extends Utils {`
`32`	`32`	`try {`
`33`	`33`	const projects = await this.gitlab.Projects.show(`${namespace}/${repoName}`);
`34`	`34`	`if (projects) {`
`35`		- console.info(`Repository with name '${repoName}' found in namespace '${namespace}'
	`35`	+ console.info(`Repository with name '${repoName}' found in organization '${namespace}'
`36`	`36`	created at '${projects.created_at}' url: gitlab.com/${namespace}/${repoName}`);
`37`	`37`	`return projects.id`
`38`	`38`	`}`
`@@ -61,7 +61,6 @@ export class GitLabProvider extends Utils {`
`61`	`61`	`await this.sleep(5000); // Wait 5 seconds before checking again`
`62`	`62`	`} catch (error) {`
`63`	`63`	`console.error('Error checking for folder creation:', error);`
`64`		`- throw error;`
`65`	`64`	`}`
`66`	`65`	`}`
`67`	`66`	`}`
`@@ -271,25 +270,21 @@ export class GitLabProvider extends Utils {`
`271`	`270`	`}`
`272`	`271`	`}`
`273`	`272`
`274`		`- public async getLatestPipeline(projectId: number) {`
`275`		`- try {`
`276`		`- // const response = await this.gitlab.Pipelines.show(projectId, 'latest');`
`277`		`- // console.log('Pipeline triggered successfully:', response);`
`278`		`- // return response;`
`279`		`-`
`280`		`- const pipelines = await this.gitlab.Pipelines.all(projectId);`
`281`		`- if (pipelines.length === 0) {`
`282`		- console.log(`No pipelines found!`);
`283`		`- return null;`
`284`		`- }`
`285`		`- const latestPipeline = pipelines.sort((a, b) => b.id - a.id)[0];`
`286`		- console.log(`Latest pipeline ID: ${latestPipeline.id} Status: ${latestPipeline.status}`);
`287`		`- return latestPipeline;`
`288`		`- } catch (error) {`
`289`		`- console.error('Error triggering pipeline:', error);`
`290`		`- throw error;`
	`273`	`+ public async getLatestPipeline(projectId: number) {`
	`274`	`+ try {`
	`275`	`+ const pipelines = await this.gitlab.Pipelines.all(projectId);`
	`276`	`+ if (pipelines.length === 0) {`
	`277`	+ console.log(`No pipelines found!`);
	`278`	`+ return null;`
`291`	`279`	`}`
	`280`	`+ const latestPipeline = pipelines.sort((a, b) => b.id - a.id)[0];`
	`281`	+ console.log(`Latest pipeline ID: ${latestPipeline.id} Status: ${latestPipeline.status}`);
	`282`	`+ return latestPipeline;`
	`283`	`+ } catch (error) {`
	`284`	`+ console.error('Error triggering pipeline:', error);`
	`285`	`+ throw error;`
`292`	`286`	`}`
	`287`	`+ }`
`293`	`288`
`294`	`289`	`// Trigger a GitLab pipeline`
`295`	`290`	`public async triggerPipeline(projectId: number, branchName: string, triggerToken: string) {`
`@@ -321,7 +316,6 @@ export class GitLabProvider extends Utils {`
`321`	`316`	`await this.sleep(5000); // Wait 5 seconds before checking again`
`322`	`317`	`} catch (error) {`
`323`	`318`	`console.error('Error checking for pipeline creation:', error);`
`324`		`- throw error;`
`325`	`319`	`}`
`326`	`320`	`}`
`327`	`321`	`}`
`@@ -346,7 +340,6 @@ export class GitLabProvider extends Utils {`
`346`	`340`	`await this.sleep(5000); // Wait 5 seconds`
`347`	`341`	`} catch (error) {`
`348`	`342`	`console.error('Error checking pipeline status:', error);`
`349`		`- throw error;`
`350`	`343`	`}`
`351`	`344`	`}`
`352`	`345`	`}`
`@@ -399,7 +392,7 @@ export class GitLabProvider extends Utils {`
`399`	`392`	`const fileContent = Buffer.from(file.content, 'base64').toString('utf-8');`
`400`	`393`
`401`	`394`	`// Replace specific text`
`402`		`- const updatedContent = fileContent.replace(textToReplace,replacement);`
	`395`	`+ const updatedContent = fileContent.replace(textToReplace, replacement);`
`403`	`396`
`404`	`397`	`// Encode the updated content to base64`
`405`	`398`	`const encodedContent = Buffer.from(updatedContent).toString('base64');`