Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add AI Core e2e tests #128

Merged
merged 19 commits into from
Sep 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions tests/e2e-tests/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
"lint:fix": "eslint . --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error"
},
"devDependencies": {
"@types/async-retry": "^1.4.8",
"async-retry": "^1.3.3",
"dotenv": "^16.4.5"
}
}
20 changes: 0 additions & 20 deletions tests/e2e-tests/src/ai-api.test.ts

This file was deleted.

146 changes: 146 additions & 0 deletions tests/e2e-tests/src/deployment-api.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import retry from 'async-retry';
import {
AiDeployment,
AiDeploymentList,
DeploymentApi
} from '@sap-ai-sdk/ai-api';
import { loadEnv } from './utils/load-env.js';
import { resourceGroup } from './utils/ai-api-utils.js';

loadEnv();

describe('DeploymentApi', () => {
let createdDeploymentId: string | undefined;
let initialState: AiDeploymentList | undefined;

beforeAll(async () => {
const queryResponse = await DeploymentApi.deploymentQuery(
{},
{ 'AI-Resource-Group': resourceGroup }
).execute();
expect(queryResponse).toBeDefined();
initialState = queryResponse;
});

it('should create a deployment and wait for it to run', async () => {
const createResponse = await DeploymentApi.deploymentCreate(
{ configurationId: '54cc966d-8bc1-44ab-a9dc-658d59ef205d' },
{ 'AI-Resource-Group': resourceGroup }
).execute();

expect(createResponse).toEqual(
expect.objectContaining({
message: 'Deployment scheduled.',
id: expect.anything()
})
);

const runningDeployment = await waitForDeploymentToReachStatus(
createResponse.id,
'RUNNING'
);

expect(runningDeployment).toEqual(
expect.objectContaining({
status: 'RUNNING',
deploymentUrl: expect.any(String)
})
);

createdDeploymentId = runningDeployment.id;
}, 180000);

it('should modify the deployment to stop it', async () => {
const deploymentId = getDeploymentId(createdDeploymentId);

const modifyResponse = await DeploymentApi.deploymentModify(
deploymentId,
{ targetStatus: 'STOPPED' },
{ 'AI-Resource-Group': resourceGroup }
).execute();

expect(modifyResponse).toEqual(
expect.objectContaining({
message: 'Deployment modification scheduled'
})
);

const stoppedDeployment = await waitForDeploymentToReachStatus(
deploymentId,
'STOPPED'
);

expect(stoppedDeployment).toEqual(
expect.objectContaining({
status: 'STOPPED'
})
);
}, 180000);

it('should delete the deployment', async () => {
const deploymentId = getDeploymentId(createdDeploymentId);
const deleteResponse = await DeploymentApi.deploymentDelete(deploymentId, {
'AI-Resource-Group': resourceGroup
}).execute();

expect(deleteResponse).toEqual(
expect.objectContaining({
message: 'Deletion scheduled'
})
);
});

afterAll(async () => {
getDeploymentId(createdDeploymentId);
// Wait for deletion to complete
await new Promise(r => setTimeout(r, 15000));
const queryResponse = await DeploymentApi.deploymentQuery(
{},
{ 'AI-Resource-Group': resourceGroup }
).execute();
expect(queryResponse).toBeDefined();

const sanitizedInitialState = sanitizedState(initialState);
const sanitizedEndState = sanitizedState(queryResponse);
expect(sanitizedEndState).toStrictEqual(sanitizedInitialState);
}, 30000);
});

const sanitizedState = (state: AiDeploymentList | undefined) => ({
...state,
resources: state?.resources.map(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
({ modifiedAt, ...rest }) => rest
)
});

function getDeploymentId(id: string | undefined): string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume the idea here is that later on this can be used to fetch a deployment ID in case it does not exist, right?

if (id === undefined) {
throw new Error('deploymentId is not defined.');
}
return id;
}

async function waitForDeploymentToReachStatus(
deploymentId: string,
targetStatus: 'RUNNING' | 'STOPPED'
): Promise<AiDeployment> {
return retry(
async () => {
const deploymentDetail = await DeploymentApi.deploymentGet(
deploymentId,
{},
{ 'AI-Resource-Group': resourceGroup }
).execute();

if (deploymentDetail.status === targetStatus) {
return deploymentDetail;
}
throw new Error(`Deployment has not yet reached ${targetStatus} status.`);
},
{
retries: 30,
minTimeout: 5000
}
);
}
9 changes: 2 additions & 7 deletions tests/e2e-tests/src/foundation-models.test.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import { chatCompletion, computeEmbedding } from '@sap-ai-sdk/sample-code';
import { loadEnv } from './utils/load-env.js';

// Pick .env file from root directory
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
dotenv.config({ path: path.resolve(__dirname, '../.env') });
loadEnv();

describe('Azure OpenAI Foundation Model Access', () => {
it('should complete a chat', async () => {
Expand Down
9 changes: 2 additions & 7 deletions tests/e2e-tests/src/orchestration.test.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import { OrchestrationClient } from '@sap-ai-sdk/orchestration';
import { loadEnv } from './utils/load-env.js';

// Pick .env file from e2e root directory
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
dotenv.config({ path: path.resolve(__dirname, '../.env') });
loadEnv();

describe('orchestration', () => {
it('should complete a chat', async () => {
Expand Down
19 changes: 19 additions & 0 deletions tests/e2e-tests/src/scenario-api.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { ScenarioApi } from '@sap-ai-sdk/ai-api';
import { loadEnv } from './utils/load-env.js';
import { resourceGroup } from './utils/ai-api-utils.js';

loadEnv();

describe('ScenarioApi', () => {
it('should get list of available scenarios', async () => {
const scenarios = await ScenarioApi.scenarioQuery({
'AI-Resource-Group': resourceGroup
}).execute();

expect(scenarios).toBeDefined();
const foundationModel = scenarios.resources.find(
scenario => scenario.id === 'foundation-models'
);
expect(foundationModel).toBeDefined();
});
});
4 changes: 4 additions & 0 deletions tests/e2e-tests/src/utils/ai-api-utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
/**
* @internal
*/
export const resourceGroup = 'ai-sdk-js-e2e';
13 changes: 13 additions & 0 deletions tests/e2e-tests/src/utils/load-env.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';

/**
* @internal
*/
export const loadEnv = (): void => {
// Pick .env file from e2e root directory
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
dotenv.config({ path: path.resolve(__dirname, '../../.env') });
};