Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvement/bb 514 handle lifecycle rules #2548

Open
wants to merge 3 commits into
base: development/8.7
Choose a base branch
from
+20 −6
Open

Improvement/bb 514 handle lifecycle rules #2548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b654e0
supported lifecycle rules in bb lifecycle config instead of const
benzekrimaha Oct 2, 2024
f953ffd
fixup post review
benzekrimaha Oct 7, 2024
45393de
fixup post review
benzekrimaha Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions extensions/lifecycle/LifecycleConfigValidator.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const {
mongoJoi,
probeServerJoi,
retryParamsJoi,
supportedLifecycleRulesJoi,
} = require('../../lib/config/configItems.joi');

const joiSchema = joi.object({
Expand Down Expand Up @@ -67,6 +68,7 @@ const joiSchema = joi.object({
coldStorageRestoreAdjustTopicPrefix: joi.string().default('cold-restore-adjust-req-'),
coldStorageGCTopicPrefix: joi.string().default('cold-gc-req-'),
coldStorageStatusTopicPrefix: joi.string().default('cold-status-'),
supportedLifecycleRules: supportedLifecycleRulesJoi,
});

function configValidator(backbeatConfig, extConfig) {
Expand Down
5 changes: 2 additions & 3 deletions extensions/lifecycle/tasks/LifecycleTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
const async = require('async');
const { errors, versioning } = require('arsenal');
const { ObjectMD } = require('arsenal').models;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to create a ticket to remove this from Arsenal, eventually
(i.e. after we have migrated all branches to this new backbeat)

Copy link
Contributor Author

@benzekrimaha benzekrimaha Oct 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ticket created here => https://scality.atlassian.net/browse/ARSN-435

const { supportedLifecycleRules } = require('arsenal').constants;
const {
LifecycleDateTime,
LifecycleUtils,
Expand Down Expand Up @@ -96,10 +95,10 @@ class LifecycleTask extends BackbeatTask {
});

this._lifecycleUtils = new LifecycleUtils(
supportedLifecycleRules,
config.extensions.lifecycle.supportedLifecycleRules,
this._lifecycleDateTime
);
this._supportedRules = supportedLifecycleRules;
this._supportedRules = config.extensions.lifecycle.supportedLifecycleRules;
this._totalRetries = 0;
}

Expand Down
4 changes: 2 additions & 2 deletions extensions/lifecycle/util/rules.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ const { RulesReducer } = require('./RulesReducer');
const { lifecycleListing: { NON_CURRENT_TYPE, CURRENT_TYPE, ORPHAN_DM_TYPE } } = require('../../../lib/constants');

const { s3middleware } = require('arsenal');
const { supportedLifecycleRules } = require('arsenal').constants;
const { scaleMsPerDay } = s3middleware.objectUtils;
const config = require('../../../lib/Config');

// Default max AWS limit is 1000 for both list objects and list object versions
const MAX_KEYS = process.env.CI === 'true' ? 3 : 1000;
Expand Down Expand Up @@ -201,7 +201,7 @@ function rulesToParams(versioningStatus, currentDate, bucketLCRules, bucketData,
* @returns {string[]} formatted supported lifecycle rules
*/
function getFormattedSupportedLifecycleRules() {
return supportedLifecycleRules.map(rule => {
return config.extensions.lifecycle.supportedLifecycleRules.map(rule => {
if (rule === 'noncurrentVersionTransition') {
return 'NoncurrentVersionTransitions';
}
Expand Down
6 changes: 6 additions & 0 deletions lib/config/configItems.joi.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const {
authTypeAccount,
authTypeService,
authTypeAssumeRole,
supportedLifecycleRules,
} = require('../constants');

const hostPortJoi = joi.object().keys({
Expand Down Expand Up @@ -150,6 +151,10 @@ const qpKafkaJoi = kafkaJoi.append({
replication: extensionKafkaJoi,
});

const supportedLifecycleRulesJoi = joi.array().items(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
const supportedLifecycleRulesJoi = joi.array().items(
const supportedLifecycleRulesJoi = joi.array().items(

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(maybe not very useful to store this one here if we only use it once, but the array can stay here)

joi.string().valid(...supportedLifecycleRules))
.default(supportedLifecycleRules);

module.exports = {
hostPortJoi,
transportJoi,
Expand All @@ -164,4 +169,5 @@ module.exports = {
stsConfigJoi,
mongoJoi,
qpKafkaJoi,
supportedLifecycleRulesJoi,
};
9 changes: 8 additions & 1 deletion lib/constants.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,14 @@ const constants = {
},
],
},
}
},
supportedLifecycleRules: [
'expiration',
'noncurrentVersionExpiration',
'abortIncompleteMultipartUpload',
'transitions',
'noncurrentVersionTransition'
]
};

module.exports = constants;
Loading