Skip to content

Add valid retry mech #3858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: v8/develop
Choose a base branch
from
Open

Add valid retry mech #3858

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
253 changes: 157 additions & 96 deletions src/service/triple-store-service.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class TripleStoreService {
// knowledgeAssetsUALs.map((ual) => `<${ual}>`),
// );
// const unifiedGraphTriples = [...triples, ...tripleAnnotations];
const promises = [];

const publicAssertion = triples.public ?? triples;

const filteredPublic = [];
Expand Down Expand Up @@ -98,26 +98,34 @@ class TripleStoreService {

const allPossibleNamedGraphs = [];

// Prepare data structures for operations
const operations = [];

if (!existsInNamedGraphs) {
promises.push(
this.tripleStoreModuleManager.createKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
publicKnowledgeAssetsUALs,
publicKnowledgeAssetsTriplesGrouped,
TRIPLES_VISIBILITY.PUBLIC,
),
);
// Public knowledge assets operations
operations.push({
name: 'createPublicKnowledgeCollectionNamedGraphs',
execute: () =>
this.tripleStoreModuleManager.createKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
publicKnowledgeAssetsUALs,
publicKnowledgeAssetsTriplesGrouped,
TRIPLES_VISIBILITY.PUBLIC,
),
});

promises.push(
this.tripleStoreModuleManager.insertMetadataTriples(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
publicKnowledgeAssetsUALs,
TRIPLES_VISIBILITY.PUBLIC,
),
);
operations.push({
name: 'insertPublicMetadataTriples',
execute: () =>
this.tripleStoreModuleManager.insertMetadataTriples(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
publicKnowledgeAssetsUALs,
TRIPLES_VISIBILITY.PUBLIC,
),
});

// current metadata triple relates to which named graph that represents Knowledge Asset hold the lates(current) data
// so for each Knowledge Asset there will be one current metadata triple
Expand Down Expand Up @@ -174,24 +182,31 @@ class TripleStoreService {
}
}

promises.push(
this.tripleStoreModuleManager.createKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
privateKnowledgeAssetsUALs,
privateKnowledgeAssetsTriplesGrouped,
TRIPLES_VISIBILITY.PRIVATE,
),
);
promises.push(
this.tripleStoreModuleManager.insertMetadataTriples(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
privateKnowledgeAssetsUALs,
TRIPLES_VISIBILITY.PRIVATE,
),
);
// Private knowledge assets operations
operations.push({
name: 'createPrivateKnowledgeCollectionNamedGraphs',
execute: () =>
this.tripleStoreModuleManager.createKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
privateKnowledgeAssetsUALs,
privateKnowledgeAssetsTriplesGrouped,
TRIPLES_VISIBILITY.PRIVATE,
),
});

operations.push({
name: 'insertPrivateMetadataTriples',
execute: () =>
this.tripleStoreModuleManager.insertMetadataTriples(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
privateKnowledgeAssetsUALs,
TRIPLES_VISIBILITY.PRIVATE,
),
});

// current metadata triple relates to which named graph that represents Knowledge Asset hold the lates(current) data
// so for each Knowledge Asset there will be one current metadata triple
// in this case there are privateKnowledgeAssetsUALs.length number of named graphs created so for each there will be one current metadata triple
Expand Down Expand Up @@ -219,15 +234,20 @@ class TripleStoreService {
}

if (paranetUAL) {
await this.tripleStoreModuleManager.createParanetKnoledgeCollectionConnection(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
paranetUAL,
contentType,
);
totalNumberOfTriplesInserted += allPossibleNamedGraphs.length; // one triple will be created for each Knowledge Asset inserted into paranet
this.logger.info(`Adding connection triples for paranet: ${paranetUAL}`);
operations.push({
name: 'createParanetConnection',
execute: async () => {
await this.tripleStoreModuleManager.createParanetKnoledgeCollectionConnection(
this.repositoryImplementations[repository],
repository,
knowledgeCollectionUAL,
paranetUAL,
contentType,
);
totalNumberOfTriplesInserted += allPossibleNamedGraphs.length; // one triple will be created for each Knowledge Asset inserted into paranet
this.logger.info(`Adding connection triples for paranet: ${paranetUAL}`);
},
});
}

// TODO: add new metadata triples and move to function insertMetadataTriples
Expand All @@ -251,78 +271,119 @@ class TripleStoreService {

totalNumberOfTriplesInserted += publicKnowledgeAssetsUALs.length + 5; // one metadata triple for each public KA

promises.push(
this.tripleStoreModuleManager.insertKnowledgeCollectionMetadata(
this.repositoryImplementations[repository],
repository,
metadataTriples,
),
);
operations.push({
name: 'insertKnowledgeCollectionMetadata',
execute: () =>
this.tripleStoreModuleManager.insertKnowledgeCollectionMetadata(
this.repositoryImplementations[repository],
repository,
metadataTriples,
),
});

const uniqueTripleCount = tripleSet.size;
totalNumberOfTriplesInserted += uniqueTripleCount;

// Implement proper retry mechanism
let attempts = 0;
let success = false;
let failedOperations = [...operations];

while (attempts < retries && !success) {
while (attempts < retries && !success && failedOperations.length > 0) {
try {
await Promise.all(promises);
success = true;

this.logger.info(
`Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`has been successfully inserted to the Triple Store's ${repository} repository.`,
);
} catch (error) {
this.logger.error(
`Error during insertion of the Knowledge Collection to the Triple Store's ${repository} repository. ` +
`UAL: ${knowledgeCollectionUAL}. Error: ${error.message}`,
// Execute all remaining operations
const currentOperations = [...failedOperations];

// Execute operations in parallel and collect results
const results = await Promise.all(
currentOperations.map(async (operation) => {
try {
await operation.execute();
return { success: true, operation };
} catch (error) {
// Return failure information instead of modifying failedOperations directly
this.logger.error(
`Operation ${operation.name} failed: ${error.message}`,
);
return { success: false, operation };
}
}),
);
attempts += 1;

if (attempts < retries) {
// Create a new array of failed operations instead of modifying the existing one
const newFailedOperations = results
.filter((result) => !result.success)
.map((result) => result.operation);

// Update failedOperations outside of any function defined in the loop
failedOperations = newFailedOperations;

// If no operations failed, we're done
if (failedOperations.length === 0) {
success = true;
this.logger.info(
`Retrying insertion of the Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository. Attempt ${
attempts + 1
} of ${retries} after delay of ${retryDelay} ms.`,
`Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`has been successfully inserted to the Triple Store's ${repository} repository.`,
);
await setTimeout(retryDelay);
} else {
this.logger.error(
`Max retries reached for the insertion of the Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository. Rolling back data.`,
);
// Some operations failed, increment attempts and retry
attempts += 1;

if (!existsInNamedGraphs) {
if (attempts < retries) {
this.logger.info(
`Rolling back Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`from the Triple Store's ${repository} repository Named Graphs.`,
`Retrying failed operations for Knowledge Collection with UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository. Attempt ${
attempts + 1
} of ${retries} after delay of ${retryDelay} ms.`,
);

await Promise.all([
this.tripleStoreModuleManager.deleteKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
allPossibleNamedGraphs,
),
this.tripleStoreModuleManager.deleteKnowledgeCollectionMetadata(
this.repositoryImplementations[repository],
repository,
allPossibleNamedGraphs,
),
]);
await setTimeout(retryDelay);
}
}
} catch (error) {
// This catch block handles any unexpected errors in the retry mechanism itself
this.logger.error(`Unexpected error during retry mechanism: ${error.message}`);
attempts += 1;

throw new Error(
`Failed to store Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository after maximum retries. Error ${error}`,
);
if (attempts < retries) {
await setTimeout(retryDelay);
}
}
}

// If we've exhausted all retries and still have failed operations
if (!success && failedOperations.length > 0) {
this.logger.error(
`Max retries reached for the insertion of the Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository. Rolling back data.`,
);

if (!existsInNamedGraphs) {
this.logger.info(
`Rolling back Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`from the Triple Store's ${repository} repository Named Graphs.`,
);

await Promise.all([
this.tripleStoreModuleManager.deleteKnowledgeCollectionNamedGraphs(
this.repositoryImplementations[repository],
repository,
allPossibleNamedGraphs,
),
this.tripleStoreModuleManager.deleteKnowledgeCollectionMetadata(
this.repositoryImplementations[repository],
repository,
allPossibleNamedGraphs,
),
]);
}

throw new Error(
`Failed to store Knowledge Collection with the UAL: ${knowledgeCollectionUAL} ` +
`to the Triple Store's ${repository} repository after maximum retries. ` +
`Failed operations: ${failedOperations.map((op) => op.name).join(', ')}`,
);
}

return totalNumberOfTriplesInserted;
}

Expand Down
Loading