Skip to content

Commit aaa5fcf

Browse files
tamirerantamireran
authored
Merge pull request #1678 from noobaa/eran_047
Support for bucket_lifecycle_configuration_rules @guymguym I couldn't find a better way to get objects list from web server process. I'm sure I'm missing something here. I will appreciate your quick review @guymguym @nimrod-becker The delete is lame. That's true in general, not just here, it will not scale well, but I took advantage of the fancy prefix mechanism to shortcut my way. We better improve it soon or at least @jackyalbo should add a test to verify how lame it is in the fancy capacity system
2 parents 79e448c + ae01498 commit aaa5fcf

File tree

11 files changed

+612
-21
lines changed

11 files changed

+612
-21
lines changed

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"dev-null": "~0.1.1",
5151
"diskspace": "~0.1.7",
5252
"dotenv": "~1.2.0",
53-
"eslint": "^1.3.1",
53+
"eslint": "2.7.0",
5454
"event-stream": "~3.3.1",
5555
"express": "~4.13.3",
5656
"forever": "^0.15.1",

‎src/api/bucket_api.js‎

Lines changed: 178 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ module.exports = {
139139
update_bucket_s3_acl: {
140140
method: 'PUT',
141141
params: {
142-
type: 'object',
142+
type: 'object',
143143
required: ['name', 'access_control'],
144144
properties: {
145145
name: {
@@ -316,8 +316,180 @@ module.exports = {
316316
auth: {
317317
system: 'admin'
318318
}
319-
}
319+
},
320+
set_bucket_lifecycle_configuration_rules: {
321+
method: 'PUT',
322+
params: {
323+
type: 'object',
324+
properties: {
325+
name: {
326+
type: 'string'
327+
},
328+
rules: {
329+
type: 'array',
330+
items: {
331+
type: 'object',
332+
required: ['id', 'prefix', 'status', 'expiration'],
333+
properties: {
334+
id: {
335+
type: 'string'
336+
},
337+
prefix: {
338+
type: 'string'
339+
},
340+
status: {
341+
type: 'string'
342+
},
343+
expiration: {
344+
type: 'object',
345+
properties: {
346+
days: {
347+
type: 'integer'
348+
},
349+
date: {
350+
format: 'idate'
351+
},
352+
expired_object_delete_marker: {
353+
type: 'boolean'
354+
}
320355

356+
}
357+
},
358+
abort_incomplete_multipart_upload: {
359+
type: 'object',
360+
properties: {
361+
days_after_initiation: {
362+
type: 'integer'
363+
},
364+
}
365+
},
366+
transition: {
367+
type: 'object',
368+
properties: {
369+
date: {
370+
format: 'idate'
371+
},
372+
storage_class: {
373+
$ref: '#/definitions/storage_class_enum'
374+
}
375+
}
376+
},
377+
noncurrent_version_expiration: {
378+
type: 'object',
379+
properties: {
380+
noncurrent_days: {
381+
type: 'integer'
382+
},
383+
}
384+
},
385+
noncurrent_version_transition: {
386+
type: 'object',
387+
properties: {
388+
noncurrent_days: {
389+
type: 'integer'
390+
},
391+
storage_class: {
392+
$ref: '#/definitions/storage_class_enum'
393+
}
394+
}
395+
},
396+
}
397+
},
398+
}
399+
}
400+
},
401+
auth: {
402+
system: 'admin'
403+
},
404+
},
405+
get_bucket_lifecycle_configuration_rules: {
406+
method: 'GET',
407+
params: {
408+
type: 'object',
409+
properties: {
410+
name: {
411+
type: 'string'
412+
}
413+
}
414+
},
415+
reply: {
416+
type: 'array',
417+
items: {
418+
type: 'object',
419+
required: ['id', 'prefix', 'status', 'expiration'],
420+
properties: {
421+
id: {
422+
type: 'string'
423+
},
424+
prefix: {
425+
type: 'string'
426+
},
427+
status: {
428+
type: 'string'
429+
},
430+
last_sync: {
431+
format: 'idate'
432+
},
433+
expiration: {
434+
type: 'object',
435+
properties: {
436+
days: {
437+
type: 'integer'
438+
},
439+
date: {
440+
format: 'idate'
441+
},
442+
expired_object_delete_marker: {
443+
type: 'boolean'
444+
}
445+
446+
}
447+
},
448+
abort_incomplete_multipart_upload: {
449+
type: 'object',
450+
properties: {
451+
days_after_initiation: {
452+
type: 'integer'
453+
},
454+
}
455+
},
456+
transition: {
457+
type: 'object',
458+
properties: {
459+
date: {
460+
format: 'idate'
461+
},
462+
storage_class: {
463+
$ref: '#/definitions/storage_class_enum'
464+
}
465+
}
466+
},
467+
noncurrent_version_expiration: {
468+
type: 'object',
469+
properties: {
470+
noncurrent_days: {
471+
type: 'integer'
472+
},
473+
}
474+
},
475+
noncurrent_version_transition: {
476+
type: 'object',
477+
properties: {
478+
noncurrent_days: {
479+
type: 'integer'
480+
},
481+
storage_class: {
482+
$ref: '#/definitions/storage_class_enum'
483+
}
484+
}
485+
},
486+
},
487+
}
488+
},
489+
auth: {
490+
system: 'admin'
491+
}
492+
},
321493
},
322494

323495
definitions: {
@@ -397,6 +569,10 @@ module.exports = {
397569
enum: ['IDLE', 'SYNCING'],
398570
type: 'string',
399571
},
572+
storage_class_enum: {
573+
enum: ['STANDARD_IA', 'GLACIER'],
574+
type: 'string'
575+
}
400576

401577
},
402578

‎src/api/object_api.js‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -589,11 +589,26 @@ module.exports = {
589589
system: 'admin'
590590
}
591591
},
592-
592+
delete_multiple_objects_by_prefix: {
593+
method: 'GET',
594+
params: {
595+
type: 'object',
596+
required: ['bucket', 'prefix'],
597+
properties: {
598+
bucket: {
599+
type: 'string',
600+
},
601+
prefix: {
602+
type: 'string',
603+
}
604+
}
605+
},
606+
auth: {
607+
system: 'admin'
608+
}
609+
},
593610
},
594611

595-
596-
597612
definitions: {
598613

599614
object_path: {

‎src/bg_workers/bg_workers_starter.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ var cloud_sync = require('./cloud_sync');
2424
var system_store = require('../server/stores/system_store');
2525
var account_server = require('../server/account_server');
2626
var scrubber = require('./scrubber');
27+
var lifecycle = require('./lifecycle');
2728
var server_rpc = require('../server/server_rpc');
2829
var dbg = require('../util/debug_module')(__filename);
2930
var db = require('../server/db');
@@ -73,5 +74,10 @@ if (process.env.SCRUBBER_DISABLED !== 'true') {
7374
building_timeout: 300000, // TODO increase?
7475
}, scrubber.background_worker);
7576
}
77+
if (process.env.LIFECYCLE_DISABLED !== 'true') {
78+
register_bg_worker({
79+
name: 'lifecycle',
80+
}, lifecycle.background_worker);
81+
}
7682

7783
dbg.log('BG Workers Server started');

‎src/bg_workers/lifecycle.js‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
'use strict';
2+
3+
module.exports = {
4+
background_worker: background_worker,
5+
};
6+
7+
var _ = require('lodash');
8+
var P = require('../util/promise');
9+
var system_store = require('../server/stores/system_store');
10+
var api = require('../api');
11+
var dbg = require('../util/debug_module')(__filename);
12+
13+
14+
var rpc = api.new_rpc();
15+
var rpc_client = rpc.new_client();
16+
17+
//TODO: remove!!! handle BG RPC better asap
18+
var S3Auth = require('aws-sdk/lib/signers/s3');
19+
var s3 = new S3Auth();
20+
21+
22+
23+
var LIFECYCLE = {
24+
schedule_min: 5 //run every 5 minutes
25+
};
26+
27+
/*
28+
*************** Cloud Sync Background Worker & Other Eports
29+
*/
30+
function background_worker() {
31+
return P.fcall(function() {
32+
dbg.log0('LIFECYCLE READ BUCKETS configuration:', 'BEGIN');
33+
return system_store.refresh()
34+
.then(function() {
35+
//TODO:
36+
//Replace asap with bg call
37+
dbg.log0('sys store', system_store.data.accounts[1]);
38+
39+
var basic_auth_params = {
40+
system: system_store.data.systems[0].name,
41+
role: 'admin'
42+
};
43+
var secret_key = system_store.data.accounts[1].access_keys[0].secret_key;
44+
var auth_params_str = JSON.stringify(basic_auth_params);
45+
var signature = s3.sign(secret_key, auth_params_str);
46+
var auth_params = {
47+
access_key: system_store.data.accounts[1].access_keys[0].access_key,
48+
string_to_sign: auth_params_str,
49+
signature: signature,
50+
};
51+
dbg.log0('create_access_key_auth', auth_params);
52+
return rpc_client.create_access_key_auth(auth_params);
53+
})
54+
.then(function() {
55+
_.each(system_store.data.buckets, function(bucket, i) {
56+
if (!bucket.lifecycle_configuration_rules) {
57+
return;
58+
}
59+
P.all(_.map(bucket.lifecycle_configuration_rules, function(lifecycle_rule, i) {
60+
dbg.log0('LIFECYCLE HANDLING BUCKET:', bucket.name, 'BEGIN');
61+
var now = Date.now();
62+
//If refresh time
63+
if (!lifecycle_rule.last_sync) {
64+
dbg.log0('LIFECYCLE HANDLING bucket:', bucket.name, 'rule id', lifecycle_rule.id, 'status:', lifecycle_rule.status, ' setting yesterday time');
65+
lifecycle_rule.last_sync = now - 1000 * 60 * 60 * 24; //set yesterday as last sync
66+
}
67+
dbg.log0('LIFECYCLE HANDLING bucket:', bucket.name, 'rule id(', i, ')', lifecycle_rule.id, 'status:', lifecycle_rule.status, 'last_sync', Math.floor((now - lifecycle_rule.last_sync) / 1000 / 60), 'min ago.');
68+
69+
if ((lifecycle_rule.status === 'Enabled') &&
70+
((now - lifecycle_rule.last_sync) / 1000 / 60 > LIFECYCLE.schedule_min)) {
71+
dbg.log0('LIFECYCLE PROCESSING bucket:', bucket.name, 'rule id(', i, ')', lifecycle_rule.id);
72+
if ((lifecycle_rule.expiration.days === 0) ||
73+
(lifecycle_rule.expiration.date < (new Date()).getTime())) {
74+
dbg.log0('LIFECYCLE DELETING bucket:', bucket.name, 'rule id(', i, ')', lifecycle_rule.id);
75+
return P.when(rpc_client.object.delete_multiple_objects_by_prefix({
76+
bucket: bucket.name,
77+
prefix: lifecycle_rule.prefix,
78+
}).then(function() {
79+
bucket.lifecycle_configuration_rules[i].last_sync = Date.now();
80+
dbg.log0('LIFECYCLE Done bucket:', bucket.name, ' done deletion of objects per prefix ', lifecycle_rule.prefix, ' time:', bucket.lifecycle_configuration_rules[i].last_sync);
81+
}));
82+
}
83+
} else {
84+
dbg.log0('LIFECYCLE NOTHING bucket:', bucket.name, 'rule id', lifecycle_rule.id, ' nothing to do');
85+
}
86+
})).then(() => {
87+
dbg.log('LIFECYCLE SYNC TIME bucket ', bucket.name, ' SAVE last sync', bucket.lifecycle_configuration_rules[0].last_sync);
88+
update_lifecycle_rules_last_sync(bucket, bucket.lifecycle_configuration_rules);
89+
});
90+
});
91+
});
92+
})
93+
.fail(function(error) {
94+
dbg.error('LIFECYCLE FAILED processing', error, error.stack);
95+
})
96+
.then(function() {
97+
dbg.log0('LIFECYCLE:', 'END');
98+
return;
99+
});
100+
}
101+
102+
103+
function update_lifecycle_rules_last_sync(bucket, rules) {
104+
return system_store.make_changes({
105+
update: {
106+
buckets: [{
107+
_id: bucket._id,
108+
lifecycle_configuration_rules: rules
109+
}]
110+
}
111+
});
112+
}

0 commit comments

Comments
 (0)