From 09240848f7fb9554d99cf889d89d8cc374f711a4 Mon Sep 17 00:00:00 2001 From: shibeshduw <166374073+shibeshduw@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:17:33 +0200 Subject: [PATCH] feat: Add AI Core e2e tests (#128) * init * Added e2e tests * increase timeout * changes to error msg * use native approach * remove unused deps * small changes * create utils * separate tests * update libs * fix lint issues * whoops --- pnpm-lock.yaml | 18 +++ tests/e2e-tests/package.json | 2 + tests/e2e-tests/src/ai-api.test.ts | 20 --- tests/e2e-tests/src/deployment-api.test.ts | 146 ++++++++++++++++++ tests/e2e-tests/src/foundation-models.test.ts | 9 +- tests/e2e-tests/src/orchestration.test.ts | 9 +- tests/e2e-tests/src/scenario-api.test.ts | 19 +++ tests/e2e-tests/src/utils/ai-api-utils.ts | 4 + tests/e2e-tests/src/utils/load-env.ts | 13 ++ 9 files changed, 206 insertions(+), 34 deletions(-) delete mode 100644 tests/e2e-tests/src/ai-api.test.ts create mode 100644 tests/e2e-tests/src/deployment-api.test.ts create mode 100644 tests/e2e-tests/src/scenario-api.test.ts create mode 100644 tests/e2e-tests/src/utils/ai-api-utils.ts create mode 100644 tests/e2e-tests/src/utils/load-env.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 07d61e88..ef20e36e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -216,6 +216,12 @@ importers: specifier: workspace:^ version: link:../../sample-code devDependencies: + '@types/async-retry': + specifier: ^1.4.8 + version: 1.4.8 + async-retry: + specifier: ^1.3.3 + version: 1.3.3 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -829,6 +835,9 @@ packages: resolution: {integrity: sha512-saiCxzHRhUrRxQV2JhH580aQUZiKQUXI38FcAcikcfOomAil4G4lxT0RfrrKywoAYP/rqAdYXYmNRLppcd+hQQ==} engines: {node: '>=14.17'} + '@types/async-retry@1.4.8': + resolution: {integrity: sha512-Qup/B5PWLe86yI5I3av6ePGaeQrIHNKCwbsQotD6aHQ6YkHsMUxVZkZsmx/Ry3VZQ6uysHwTjQ7666+k6UjVJA==} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -913,6 +922,9 @@ packages: '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + '@types/retry@0.12.5': + resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + '@types/semver@7.5.8': resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} @@ -4668,6 +4680,10 @@ snapshots: '@tsd/typescript@5.4.5': {} + '@types/async-retry@1.4.8': + dependencies: + '@types/retry': 0.12.5 + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.25.3 @@ -4774,6 +4790,8 @@ snapshots: '@types/range-parser@1.2.7': {} + '@types/retry@0.12.5': {} + '@types/semver@7.5.8': {} '@types/send@0.17.4': diff --git a/tests/e2e-tests/package.json b/tests/e2e-tests/package.json index decec382..63295a28 100644 --- a/tests/e2e-tests/package.json +++ b/tests/e2e-tests/package.json @@ -25,6 +25,8 @@ "lint:fix": "eslint . --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error" }, "devDependencies": { + "@types/async-retry": "^1.4.8", + "async-retry": "^1.3.3", "dotenv": "^16.4.5" } } diff --git a/tests/e2e-tests/src/ai-api.test.ts b/tests/e2e-tests/src/ai-api.test.ts deleted file mode 100644 index 71148f17..00000000 --- a/tests/e2e-tests/src/ai-api.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import path from 'path'; -import { fileURLToPath } from 'url'; -import dotenv from 'dotenv'; -import { DeploymentApi } from '@sap-ai-sdk/ai-api'; -import 'dotenv/config'; - -// Pick .env file from root directory -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -dotenv.config({ path: path.resolve(__dirname, '../.env') }); - -describe('ai-api', () => { - it('should get deployments', async () => { - const deployments = await DeploymentApi.deploymentQuery( - {}, - { 'AI-Resource-Group': 'default' } - ).execute(); - expect(deployments).toBeDefined(); - }); -}); diff --git a/tests/e2e-tests/src/deployment-api.test.ts b/tests/e2e-tests/src/deployment-api.test.ts new file mode 100644 index 00000000..8188ea0f --- /dev/null +++ b/tests/e2e-tests/src/deployment-api.test.ts @@ -0,0 +1,146 @@ +import retry from 'async-retry'; +import { + AiDeployment, + AiDeploymentList, + DeploymentApi +} from '@sap-ai-sdk/ai-api'; +import { loadEnv } from './utils/load-env.js'; +import { resourceGroup } from './utils/ai-api-utils.js'; + +loadEnv(); + +describe('DeploymentApi', () => { + let createdDeploymentId: string | undefined; + let initialState: AiDeploymentList | undefined; + + beforeAll(async () => { + const queryResponse = await DeploymentApi.deploymentQuery( + {}, + { 'AI-Resource-Group': resourceGroup } + ).execute(); + expect(queryResponse).toBeDefined(); + initialState = queryResponse; + }); + + it('should create a deployment and wait for it to run', async () => { + const createResponse = await DeploymentApi.deploymentCreate( + { configurationId: '54cc966d-8bc1-44ab-a9dc-658d59ef205d' }, + { 'AI-Resource-Group': resourceGroup } + ).execute(); + + expect(createResponse).toEqual( + expect.objectContaining({ + message: 'Deployment scheduled.', + id: expect.anything() + }) + ); + + const runningDeployment = await waitForDeploymentToReachStatus( + createResponse.id, + 'RUNNING' + ); + + expect(runningDeployment).toEqual( + expect.objectContaining({ + status: 'RUNNING', + deploymentUrl: expect.any(String) + }) + ); + + createdDeploymentId = runningDeployment.id; + }, 180000); + + it('should modify the deployment to stop it', async () => { + const deploymentId = getDeploymentId(createdDeploymentId); + + const modifyResponse = await DeploymentApi.deploymentModify( + deploymentId, + { targetStatus: 'STOPPED' }, + { 'AI-Resource-Group': resourceGroup } + ).execute(); + + expect(modifyResponse).toEqual( + expect.objectContaining({ + message: 'Deployment modification scheduled' + }) + ); + + const stoppedDeployment = await waitForDeploymentToReachStatus( + deploymentId, + 'STOPPED' + ); + + expect(stoppedDeployment).toEqual( + expect.objectContaining({ + status: 'STOPPED' + }) + ); + }, 180000); + + it('should delete the deployment', async () => { + const deploymentId = getDeploymentId(createdDeploymentId); + const deleteResponse = await DeploymentApi.deploymentDelete(deploymentId, { + 'AI-Resource-Group': resourceGroup + }).execute(); + + expect(deleteResponse).toEqual( + expect.objectContaining({ + message: 'Deletion scheduled' + }) + ); + }); + + afterAll(async () => { + getDeploymentId(createdDeploymentId); + // Wait for deletion to complete + await new Promise(r => setTimeout(r, 15000)); + const queryResponse = await DeploymentApi.deploymentQuery( + {}, + { 'AI-Resource-Group': resourceGroup } + ).execute(); + expect(queryResponse).toBeDefined(); + + const sanitizedInitialState = sanitizedState(initialState); + const sanitizedEndState = sanitizedState(queryResponse); + expect(sanitizedEndState).toStrictEqual(sanitizedInitialState); + }, 30000); +}); + +const sanitizedState = (state: AiDeploymentList | undefined) => ({ + ...state, + resources: state?.resources.map( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ({ modifiedAt, ...rest }) => rest + ) +}); + +function getDeploymentId(id: string | undefined): string { + if (id === undefined) { + throw new Error('deploymentId is not defined.'); + } + return id; +} + +async function waitForDeploymentToReachStatus( + deploymentId: string, + targetStatus: 'RUNNING' | 'STOPPED' +): Promise { + return retry( + async () => { + const deploymentDetail = await DeploymentApi.deploymentGet( + deploymentId, + {}, + { 'AI-Resource-Group': resourceGroup } + ).execute(); + + if (deploymentDetail.status === targetStatus) { + return deploymentDetail; + } + throw new Error(`Deployment has not yet reached ${targetStatus} status.`); + }, + { + retries: 30, + minTimeout: 5000 + } + ); +} diff --git a/tests/e2e-tests/src/foundation-models.test.ts b/tests/e2e-tests/src/foundation-models.test.ts index 8909fc3c..a2ad3dd4 100644 --- a/tests/e2e-tests/src/foundation-models.test.ts +++ b/tests/e2e-tests/src/foundation-models.test.ts @@ -1,12 +1,7 @@ -import path from 'path'; -import { fileURLToPath } from 'url'; -import dotenv from 'dotenv'; import { chatCompletion, computeEmbedding } from '@sap-ai-sdk/sample-code'; +import { loadEnv } from './utils/load-env.js'; -// Pick .env file from root directory -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -dotenv.config({ path: path.resolve(__dirname, '../.env') }); +loadEnv(); describe('Azure OpenAI Foundation Model Access', () => { it('should complete a chat', async () => { diff --git a/tests/e2e-tests/src/orchestration.test.ts b/tests/e2e-tests/src/orchestration.test.ts index 16fdce50..daeebfed 100644 --- a/tests/e2e-tests/src/orchestration.test.ts +++ b/tests/e2e-tests/src/orchestration.test.ts @@ -1,12 +1,7 @@ -import path from 'path'; -import { fileURLToPath } from 'url'; -import dotenv from 'dotenv'; import { OrchestrationClient } from '@sap-ai-sdk/orchestration'; +import { loadEnv } from './utils/load-env.js'; -// Pick .env file from e2e root directory -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -dotenv.config({ path: path.resolve(__dirname, '../.env') }); +loadEnv(); describe('orchestration', () => { it('should complete a chat', async () => { diff --git a/tests/e2e-tests/src/scenario-api.test.ts b/tests/e2e-tests/src/scenario-api.test.ts new file mode 100644 index 00000000..2ee37eff --- /dev/null +++ b/tests/e2e-tests/src/scenario-api.test.ts @@ -0,0 +1,19 @@ +import { ScenarioApi } from '@sap-ai-sdk/ai-api'; +import { loadEnv } from './utils/load-env.js'; +import { resourceGroup } from './utils/ai-api-utils.js'; + +loadEnv(); + +describe('ScenarioApi', () => { + it('should get list of available scenarios', async () => { + const scenarios = await ScenarioApi.scenarioQuery({ + 'AI-Resource-Group': resourceGroup + }).execute(); + + expect(scenarios).toBeDefined(); + const foundationModel = scenarios.resources.find( + scenario => scenario.id === 'foundation-models' + ); + expect(foundationModel).toBeDefined(); + }); +}); diff --git a/tests/e2e-tests/src/utils/ai-api-utils.ts b/tests/e2e-tests/src/utils/ai-api-utils.ts new file mode 100644 index 00000000..b45ef62d --- /dev/null +++ b/tests/e2e-tests/src/utils/ai-api-utils.ts @@ -0,0 +1,4 @@ +/** + * @internal + */ +export const resourceGroup = 'ai-sdk-js-e2e'; diff --git a/tests/e2e-tests/src/utils/load-env.ts b/tests/e2e-tests/src/utils/load-env.ts new file mode 100644 index 00000000..38f5aa04 --- /dev/null +++ b/tests/e2e-tests/src/utils/load-env.ts @@ -0,0 +1,13 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; +import dotenv from 'dotenv'; + +/** + * @internal + */ +export const loadEnv = (): void => { + // Pick .env file from e2e root directory + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + dotenv.config({ path: path.resolve(__dirname, '../../.env') }); +};