Skip to content

Commit 4d051bf

Browse files
timofey-barmintimofey-barmin
committed
MB-61292: Permanently remove retired keys after 3 months
Change-Id: I117175202e6d88f90a1d0fbd71bcb85284aafb79 Reviewed-on: https://review.couchbase.org/c/ns_server/+/221116 Reviewed-by: Navdeep S Boparai <[email protected]> Tested-by: Timofey Barmin <[email protected]> Well-Formed: Build Bot <[email protected]>
1 parent 195ae9b commit 4d051bf

File tree

2 files changed

+213
-6
lines changed

2 files changed

+213
-6
lines changed

‎apps/ns_server/src/cb_cluster_secrets.erl

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@
9292
rotate_keks => undefined,
9393
dek_cleanup => undefined,
9494
rotate_deks => undefined,
95-
dek_info_update => undefined}
95+
dek_info_update => undefined,
96+
remove_retired_keys => undefined}
9697
:: #{atom() := reference() | undefined},
9798
deks = undefined :: #{cb_deks:dek_kind() := deks_info()} |
9899
undefined,
@@ -707,7 +708,8 @@ init([Type]) ->
707708
run_jobs(_),
708709
restart_dek_cleanup_timer(_),
709710
restart_rotation_timer(_),
710-
restart_dek_info_update_timer(true, _)])}.
711+
restart_dek_info_update_timer(true, _),
712+
restart_remove_retired_timer(_)])}.
711713

712714
handle_call({call, {M, F, A} = MFA}, _From,
713715
#state{proc_type = ?MASTER_PROC} = State) ->
@@ -861,6 +863,13 @@ handle_info({timer, dek_info_update} = Msg,
861863
{_Res, NewState} = calculate_dek_info(State),
862864
{noreply, restart_dek_info_update_timer(false, NewState)};
863865

866+
handle_info({timer, remove_retired_keys} = Msg,
867+
#state{proc_type = ?NODE_PROC} = State) ->
868+
?log_debug("Remove retired keys timer"),
869+
misc:flush(Msg),
870+
encryption_service:cleanup_retired_keys(),
871+
{noreply, restart_remove_retired_timer(State)};
872+
864873
handle_info(Info, State) ->
865874
?log_warning("Unhandled info: ~p", [Info]),
866875
{noreply, State}.
@@ -1607,6 +1616,7 @@ maybe_read_deks(#state{proc_type = ?NODE_PROC, deks = undefined} = State) ->
16071616
NewAcc
16081617
end, NewState2, Kinds),
16091618
ok = garbage_collect_keks(),
1619+
encryption_service:cleanup_retired_keys(),
16101620

16111621
%% We should rotate here (when process is starting) because we can
16121622
%% hypothetically crash after calling set_active() but before calling
@@ -2445,6 +2455,29 @@ dek_rotation_time(Kind, #{is_enabled := true, active_id := ActiveId,
24452455
{error, E}
24462456
end.
24472457

2458+
-spec calculate_next_remove_retired_time(calendar:datetime()) ->
2459+
non_neg_integer().
2460+
calculate_next_remove_retired_time({{Year, Month, _Day},
2461+
{_Hour, _Min, _Sec}} = Now) ->
2462+
%% We want to remove retired keys at 12:00:00 of the first day of next month
2463+
%% Calculate first day of next month
2464+
{NextYear, NextMonth} =
2465+
case Month of
2466+
12 -> {Year + 1, 1};
2467+
_ -> {Year, Month + 1}
2468+
end,
2469+
NextTime = {{NextYear, NextMonth, 1}, {12, 0, 0}},
2470+
CurrentSecs = calendar:datetime_to_gregorian_seconds(Now),
2471+
NextSecs = calendar:datetime_to_gregorian_seconds(NextTime),
2472+
(NextSecs - CurrentSecs) * 1000.
2473+
2474+
-spec restart_remove_retired_timer(#state{}) -> #state{}.
2475+
restart_remove_retired_timer(#state{proc_type = ?MASTER_PROC} = State) ->
2476+
State;
2477+
restart_remove_retired_timer(#state{proc_type = ?NODE_PROC} = State) ->
2478+
Time = calculate_next_remove_retired_time(calendar:universal_time()),
2479+
restart_timer(remove_retired_keys, Time, State).
2480+
24482481
validate_for_config_encryption(#{type := T,
24492482
data := #{encrypt_by := nodeSecretManager}},
24502483
Snapshot) when T == ?GENERATED_KEY_TYPE;
@@ -3305,4 +3338,39 @@ update_next_rotation_time_test() ->
33053338
?assertEqual(Future(3 * D - 1), Calc(Past(12 * D + 1), 3, true)),
33063339
?assertEqual(Future(1), Calc(Past(12 * D - 1), 3, true)).
33073340

3341+
calculate_next_remove_retired_time_test() ->
3342+
TimeDiff = fun (A, B) ->
3343+
(calendar:datetime_to_gregorian_seconds(A) -
3344+
calendar:datetime_to_gregorian_seconds(B)) * 1000
3345+
end,
3346+
%% Test mid-month case
3347+
MidMonth = {{2023, 6, 15}, {14, 30, 45}},
3348+
ExpectedMidMonth = TimeDiff({{2023, 7, 1}, {12, 0, 0}}, MidMonth),
3349+
?assertEqual(ExpectedMidMonth,
3350+
calculate_next_remove_retired_time(MidMonth)),
3351+
3352+
%% Test end of month case
3353+
EndMonth = {{2023, 6, 30}, {23, 59, 59}},
3354+
ExpectedEndMonth = TimeDiff({{2023, 7, 1}, {12, 0, 0}}, EndMonth),
3355+
?assertEqual(ExpectedEndMonth,
3356+
calculate_next_remove_retired_time(EndMonth)),
3357+
3358+
%% Test start of month case
3359+
StartMonth = {{2023, 6, 1}, {0, 0, 0}},
3360+
ExpectedStartMonth = TimeDiff({{2023, 7, 1}, {12, 0, 0}}, StartMonth),
3361+
?assertEqual(ExpectedStartMonth,
3362+
calculate_next_remove_retired_time(StartMonth)),
3363+
3364+
%% Test December case (year rollover)
3365+
December = {{2023, 12, 15}, {12, 0, 0}},
3366+
ExpectedDecember = TimeDiff({{2024, 1, 1}, {12, 0, 0}}, December),
3367+
?assertEqual(ExpectedDecember,
3368+
calculate_next_remove_retired_time(December)),
3369+
3370+
%% Test exactly at noon on first of month
3371+
AtNoon = {{2023, 6, 1}, {12, 0, 0}},
3372+
ExpectedAtNoon = TimeDiff({{2023, 7, 1}, {12, 0, 0}}, AtNoon),
3373+
?assertEqual(ExpectedAtNoon,
3374+
calculate_next_remove_retired_time(AtNoon)).
3375+
33083376
-endif.

‎apps/ns_server/src/encryption_service.erl

Lines changed: 143 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
decode_key_info/1,
4040
garbage_collect_keks/1,
4141
garbage_collect_keys/2,
42+
cleanup_retired_keys/0,
4243
maybe_rotate_integrity_tokens/1,
4344
remove_old_integrity_tokens/1,
4445
get_key_ids_in_use/0]).
@@ -55,6 +56,8 @@
5556

5657
-define(RUNNER, {cb_gosecrets_runner, ns_server:get_babysitter_node()}).
5758
-define(RESTART_WAIT_TIMEOUT, 120000).
59+
-define(RETIRED_KEYS_RETENTION_MONTHS,
60+
?get_param(retired_keys_retention_months, 3)).
5861

5962
start_link() ->
6063
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
@@ -562,8 +565,144 @@ select_files_to_retire(Dir, _KeyIdsInUse, {error, Reason}) ->
562565
[Dir, Reason]),
563566
[].
564567

568+
cleanup_retired_keys() ->
569+
cleanup_retired_keys(calendar:universal_time(),
570+
?RETIRED_KEYS_RETENTION_MONTHS).
571+
572+
cleanup_retired_keys({{CurrentYear, CurrentMonth, _}, _}, RetentionMonths) ->
573+
?log_debug("Cleanup retired keys"),
574+
RetiredDir = retired_keys_dir(),
575+
maybe
576+
{ok, Dirs} ?= file:list_dir(RetiredDir),
577+
CurrentMonths = CurrentYear * 12 + CurrentMonth,
578+
lists:foreach(
579+
fun (DirName) ->
580+
FullPath = filename:join(RetiredDir, DirName),
581+
maybe
582+
{dir, true} ?= {dir, filelib:is_dir(FullPath)},
583+
{ok, {Year, Month}} ?= parse_retired_dir_name(DirName),
584+
DirMonths = Year * 12 + Month,
585+
{old, true} ?=
586+
{old, CurrentMonths - DirMonths > RetentionMonths},
587+
AllFiles = file:list_dir(FullPath),
588+
?log_info("Permanently removing retired keys directory ~s as "
589+
"it is older than ~b months. Keys to be "
590+
"removed: ~.0p",
591+
[FullPath, RetentionMonths, AllFiles]),
592+
ok ?= misc:rm_rf(FullPath)
593+
else
594+
{dir, false} ->
595+
?log_warning("Invalid retired keys directory name "
596+
"(not a directory): ~s, will be "
597+
"ignored", [FullPath]);
598+
error ->
599+
?log_warning("Invalid retired keys directory name: ~s, "
600+
"will be ignored", [DirName]);
601+
{old, false} ->
602+
ok;
603+
{error, Reason} ->
604+
?log_error("Failed to remove retired keys directory ~s: "
605+
"~p", [FullPath, Reason])
606+
end
607+
end, Dirs),
608+
ok
609+
else
610+
{error, enoent} ->
611+
ok;
612+
{error, Reason} ->
613+
?log_error("Failed to list retired keys directory ~s: ~p",
614+
[RetiredDir, Reason]),
615+
ok
616+
end.
617+
618+
parse_retired_dir_name(DirName) ->
619+
try
620+
[YearStr, MonthStr] = string:tokens(DirName, "-"),
621+
Year = list_to_integer(YearStr),
622+
Month = list_to_integer(MonthStr),
623+
case calendar:valid_date(Year, Month, 1) of
624+
true -> {ok, {Year, Month}};
625+
false -> error
626+
end
627+
catch
628+
_:_ -> error
629+
end.
630+
565631
-ifdef(TEST).
566632

633+
cleanup_retired_keys_test() ->
634+
%% Create temp directory for test
635+
RetiredDir = retired_keys_dir(),
636+
ok = filelib:ensure_dir(RetiredDir ++ "/"),
637+
638+
%% Helper to create test directories and files
639+
CreateTestDir =
640+
fun (YearMonth) ->
641+
Dir = filename:join(RetiredDir, YearMonth),
642+
ok = filelib:ensure_dir(Dir ++ "/"),
643+
ok = file:write_file(filename:join(Dir, "test.key.1"), <<"test">>)
644+
end,
645+
646+
try
647+
%% Create test directories for different months
648+
lists:foreach(CreateTestDir, [
649+
"2023-10", %% Should be removed (>3 months old)
650+
"2023-11", %% Should be removed (>3 months old)
651+
"2023-12", %% Should stay (3 months old)
652+
"2024-01", %% Should stay (2 months old)
653+
"2025-02" %% Should stay (in future)
654+
]),
655+
656+
%% Also create some invalid directory names that should be ignored
657+
lists:foreach(CreateTestDir, [
658+
"not-a-date",
659+
"2023-13",
660+
"2023-0",
661+
"2023"
662+
]),
663+
%% Create a file in retiredKeysDir, should be ignored
664+
ok = file:write_file(filename:join(RetiredDir, "test.key.1"),
665+
<<"test">>),
666+
667+
%% Run cleanup with reference date of 2024-03-01 and 3 month retention
668+
ok = cleanup_retired_keys({{2024, 3, 1}, {0,0,0}}, 3),
669+
670+
%% Verify correct directories were removed/kept
671+
?assertNot(filelib:is_dir(filename:join(RetiredDir, "2023-10"))),
672+
?assertNot(filelib:is_dir(filename:join(RetiredDir, "2023-11"))),
673+
?assert(filelib:is_dir(filename:join(RetiredDir, "2023-12"))),
674+
?assert(filelib:is_dir(filename:join(RetiredDir, "2024-01"))),
675+
?assert(filelib:is_dir(filename:join(RetiredDir, "2025-02"))),
676+
677+
%% Invalid directories and files should still exist since they were
678+
%% ignored
679+
?assert(filelib:is_dir(filename:join(RetiredDir, "not-a-date"))),
680+
?assert(filelib:is_dir(filename:join(RetiredDir, "2023-13"))),
681+
?assert(filelib:is_dir(filename:join(RetiredDir, "2023-0"))),
682+
?assert(filelib:is_dir(filename:join(RetiredDir, "2023"))),
683+
?assert(filelib:is_file(filename:join(RetiredDir, "test.key.1")))
684+
685+
after
686+
%% Cleanup test directory
687+
ok = misc:rm_rf(RetiredDir)
688+
end.
689+
690+
parse_retired_dir_name_test() ->
691+
?assertEqual({ok, {2023, 12}}, parse_retired_dir_name("2023-12")),
692+
?assertEqual({ok, {2024, 1}}, parse_retired_dir_name("2024-1")),
693+
?assertEqual({ok, {2024, 1}}, parse_retired_dir_name("2024-01")),
694+
?assertEqual(error, parse_retired_dir_name("2024-13")),
695+
?assertEqual(error, parse_retired_dir_name("2024-0")),
696+
?assertEqual(error, parse_retired_dir_name("2024")),
697+
?assertEqual(error, parse_retired_dir_name("2024-")),
698+
?assertEqual(error, parse_retired_dir_name("2024-")),
699+
?assertEqual(error, parse_retired_dir_name("-12")),
700+
?assertEqual(error, parse_retired_dir_name("abc-12")),
701+
?assertEqual(error, parse_retired_dir_name("2024-abc")),
702+
?assertEqual(error, parse_retired_dir_name("")),
703+
?assertEqual(error, parse_retired_dir_name("2024-12-25")).
704+
705+
567706
-define(A, "a0000000-0000-0000-0000-000000000000").
568707
-define(B, "b0000000-0000-0000-0000-000000000000").
569708
-define(C, "c0000000-0000-0000-0000-000000000000").
@@ -630,15 +769,15 @@ key_path(Kind) ->
630769
bucket_dek_id(Bucket, DekId) ->
631770
iolist_to_binary(filename:join([Bucket, "deks", DekId])).
632771

772+
retired_keys_dir() ->
773+
filename:join(path_config:component_path(data), "retired_keys").
774+
633775
retire_key(Kind, Filename) ->
634776
Dir = key_path(Kind),
635777
FromPath = filename:join(Dir, Filename),
636778
{{Y, M, _}, _} = calendar:universal_time(),
637779
MonthDir = lists:flatten(io_lib:format("~b-~b", [Y, M])),
638-
ToPath = filename:join([path_config:component_path(data),
639-
"retired_keys",
640-
MonthDir,
641-
Filename]),
780+
ToPath = filename:join([retired_keys_dir(), MonthDir, Filename]),
642781
case filelib:ensure_dir(ToPath) of
643782
ok ->
644783
case misc:atomic_rename(FromPath, ToPath) of

0 commit comments

Comments
 (0)