Skip to content

Commit

Permalink
implement caching without full model explicitly
Browse files Browse the repository at this point in the history
  • Loading branch information
marikaner committed Sep 3, 2024
1 parent 2959a43 commit 8ececb3
Showing 1 changed file with 33 additions and 15 deletions.
48 changes: 33 additions & 15 deletions packages/gen-ai-hub/src/utils/deployment-cache.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Cache } from '@sap-cloud-sdk/connectivity/internal.js';
import { type AiDeployment } from '@sap-ai-sdk/ai-core';
import { unique } from '@sap-cloud-sdk/util';
import { type DeploymentResolutionOptions } from './deployment-resolver.js';
import { extractModel, type FoundationModel } from './model.js';

Expand Down Expand Up @@ -54,29 +55,46 @@ function createDeploymentCache(cache: Cache<Deployment>) {
opts: DeploymentResolutionOptions,
deployments: AiDeployment[]
): void => {
// go backwards to cache the first deployment ID for each model
[...deployments].reverse().forEach(deployment => {
const model = extractModel(deployment);
const entry = transformDeploymentForCache(deployment);
cache.set(getCacheKey({ ...opts, model }), {
const cacheEntries = [...deployments].map(deployment =>
transformDeploymentForCache(deployment)
);

[
...unique(cacheEntries),
...getCacheEntriesForModelsWithoutVersion(cacheEntries),
...getCacheEntriesForNoModel(cacheEntries)
].forEach(entry => {
cache.set(getCacheKey({ ...opts, model: entry.model }), {
entry
});

if (model) {
cache.set(getCacheKey({ ...opts, model: undefined }), { entry });
if (model.version) {
cache.set(
getCacheKey({ ...opts, model: { ...model, version: undefined } }),
{ entry }
);
}
}
});
},
clear: () => cache.clear()
};
}

function getCacheEntriesForModelsWithoutVersion(
cacheEntries: Deployment[]
): Deployment[] {
const modelNames = unique(
cacheEntries
.filter(({ model }) => model?.name && model.version)
.map(({ model }) => model?.name as string)
);

return modelNames.map(modelName => ({
// cannot be undefined
id: cacheEntries.find(({ model }) => model?.name === modelName)!.id,
model: { name: modelName, version: undefined }
}));
}

function getCacheEntriesForNoModel(cacheEntries: Deployment[]): Deployment[] {
return cacheEntries[0]?.model
? [{ ...cacheEntries[0], model: undefined }]
: [];
}

function transformDeploymentForCache(deployment: AiDeployment): Deployment {
return {
id: deployment.id,
Expand Down

0 comments on commit 8ececb3

Please sign in to comment.