Skip to content

Commit

Permalink
support incremental backups
Browse files Browse the repository at this point in the history
  • Loading branch information
@posriniv
posriniv committed Jan 25, 2024
1 parent 5fcf68d commit 7f2f9c5
Show file tree
Hide file tree
Showing 9 changed files with 109 additions and 126 deletions.
85 changes: 32 additions & 53 deletions cmd/backup/policy/policy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/yugabyte/ybm-cli/cmd/util"
ybmAuthClient "github.com/yugabyte/ybm-cli/internal/client"
"github.com/yugabyte/ybm-cli/internal/formatter"
ybmclient "github.com/yugabyte/yugabytedb-managed-go-client-internal"
Expand Down Expand Up @@ -113,29 +114,15 @@ var enablePolicyCmd = &cobra.Command{
if len(resp.GetData()) < 1 {
logrus.Fatalln("No backup policies found for the given cluster")
}
scheduleSpec := resp.GetData()[0].GetSpec()
if scheduleSpec.GetState() == ybmclient.SCHEDULESTATEENUM_ACTIVE {
backupScheduleSpec := resp.GetData()[0].GetSpec()
if backupScheduleSpec.GetState() == ybmclient.SCHEDULESTATEENUM_ACTIVE {
logrus.Fatalf("The backup policy is already enabled for cluster %s\n", formatter.Colorize(clusterName, formatter.GREEN_COLOR))
}
scheduleSpec.SetState(ybmclient.SCHEDULESTATEENUM_ACTIVE)
backupScheduleSpec.SetState(ybmclient.SCHEDULESTATEENUM_ACTIVE)
info := resp.GetData()[0].GetInfo()
scheduleId := info.GetId()
retentionPeriodInDaysResp, retentionFound := info.GetTaskParams()["retention_period_in_days"]
if !retentionFound || retentionPeriodInDaysResp == nil {
logrus.Fatalln("Unable to fetch retention period in days for the backup schedule")
}
retentionPeriodInDays := int32(retentionPeriodInDaysResp.(float64))
descriptionResp, descriptionFound := info.GetTaskParams()["description"]
if !descriptionFound || descriptionResp == nil {
logrus.Fatalln("Unable to fetch description for the backup schedule")
}
description := descriptionResp.(string)
backupSpec := ybmclient.NewBackupSpec(clusterId)
backupSpec.SetRetentionPeriodInDays(retentionPeriodInDays)
backupSpec.SetDescription(description)

backupScheduleSpec := ybmclient.NewBackupScheduleSpec(*backupSpec, scheduleSpec)
_, r, err = authApi.UpdateBackupPolicy(scheduleId).BackupScheduleSpec(*backupScheduleSpec).Execute()
_, r, err = authApi.UpdateBackupPolicy(clusterId, scheduleId).ScheduleSpecV2(backupScheduleSpec).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
Expand All @@ -162,7 +149,7 @@ var disablePolicyCmd = &cobra.Command{
logrus.Fatal(err)
}

listBackupPoliciesRequest := authApi.ListBackupPolicies(clusterId, false /* fetchOnlyActive */)
listBackupPoliciesRequest := authApi.ListBackupPolicies(clusterId, true /* fetchOnlyActive */)

resp, r, err := listBackupPoliciesRequest.Execute()
if err != nil {
Expand All @@ -173,35 +160,19 @@ var disablePolicyCmd = &cobra.Command{
if len(resp.GetData()) < 1 {
logrus.Fatalln("No backup policies found for the given cluster")
}
scheduleSpec := resp.GetData()[0].GetSpec()
if scheduleSpec.GetState() == ybmclient.SCHEDULESTATEENUM_PAUSED {
backupScheduleSpec := resp.GetData()[0].GetSpec()
if backupScheduleSpec.GetState() == ybmclient.SCHEDULESTATEENUM_PAUSED {
logrus.Fatalf("The backup policy is already disabled for cluster %s\n", formatter.Colorize(clusterName, formatter.GREEN_COLOR))
}
scheduleSpec.SetState(ybmclient.SCHEDULESTATEENUM_PAUSED)
backupScheduleSpec.SetState(ybmclient.SCHEDULESTATEENUM_PAUSED)
info := resp.GetData()[0].GetInfo()
scheduleId := info.GetId()
retentionPeriodInDaysResp, retentionFound := info.GetTaskParams()["retention_period_in_days"]
if !retentionFound || retentionPeriodInDaysResp == nil {
logrus.Fatalln("Unable to fetch retention period in days for the backup schedule")
}
retentionPeriodInDays := int32(retentionPeriodInDaysResp.(float64))
descriptionResp, descriptionFound := info.GetTaskParams()["description"]
if !descriptionFound || descriptionResp == nil {
logrus.Fatalln("Unable to fetch description for the backup schedule")
return
}
description := descriptionResp.(string)
backupSpec := ybmclient.NewBackupSpec(clusterId)
backupSpec.SetRetentionPeriodInDays(retentionPeriodInDays)
backupSpec.SetDescription(description)

backupScheduleSpec := ybmclient.NewBackupScheduleSpec(*backupSpec, scheduleSpec)
_, r, err = authApi.UpdateBackupPolicy(scheduleId).BackupScheduleSpec(*backupScheduleSpec).Execute()
_, r, err = authApi.UpdateBackupPolicy(clusterId, scheduleId).ScheduleSpecV2(backupScheduleSpec).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}

fmt.Printf("Successfully disabled backup policy for cluster %s\n", formatter.Colorize(clusterName, formatter.GREEN_COLOR))

},
Expand Down Expand Up @@ -238,14 +209,18 @@ var updatePolicyCmd = &cobra.Command{
if len(resp.GetData()) < 1 {
logrus.Fatalln("No backup policies found for the given cluster")
}
scheduleSpec := resp.GetData()[0].GetSpec()

info := resp.GetData()[0].GetInfo()
scheduleId := info.GetId()
backupScheduleSpec := resp.GetData()[0].GetSpec()

if cmd.Flags().Changed("full-backup-frequency-in-days") {
frequencyInDays, _ := cmd.Flags().GetInt32("full-backup-frequency-in-days")
if frequencyInDays < 1 {
logrus.Fatalln("Time interval for scheduling backup should be greater than or equal to 1 day")
}
scheduleSpec.SetTimeIntervalInDays(frequencyInDays)
scheduleSpec.UnsetCronExpression()
backupScheduleSpec.SetTimeIntervalInDays(frequencyInDays)
backupScheduleSpec.UnsetCronExpression()
} else {
daysOfWeek, _ := cmd.Flags().GetString("full-backup-schedule-days-of-week")
if !isDaysOfWeekValid(daysOfWeek) {
Expand All @@ -257,19 +232,21 @@ var updatePolicyCmd = &cobra.Command{
}
backupTimeUTC := convertLocalTimeToUTC(backupTime)
cronExpression := generateCronExpression(daysOfWeek, backupTimeUTC)
scheduleSpec.SetCronExpression(cronExpression)
scheduleSpec.TimeIntervalInDays = nil
backupScheduleSpec.SetCronExpression(cronExpression)
backupScheduleSpec.TimeIntervalInDays = nil
}

info := resp.GetData()[0].GetInfo()
scheduleId := info.GetId()
description := info.GetTaskParams()["description"].(string)
backupSpec := ybmclient.NewBackupSpec(clusterId)
backupSpec.SetRetentionPeriodInDays(retentionPeriodInDays)
backupSpec.SetDescription(description)
if util.IsFeatureFlagEnabled(util.INCREMENTAL_BACKUP) {
if cmd.Flags().Changed("incremental-backup-frequency-in-minutes") {
incrementalBackupFrequencyInMinutes, _ := cmd.Flags().GetInt32("incremental-backup-frequency-in-minutes")
if incrementalBackupFrequencyInMinutes < 1 {
logrus.Fatalln("Time interval for scheduling incremental backup cannot be negative or zero")
}
backupScheduleSpec.SetIncrementalIntervalInMinutes(incrementalBackupFrequencyInMinutes)
}
}

backupScheduleSpec := ybmclient.NewBackupScheduleSpec(*backupSpec, scheduleSpec)
_, r, err = authApi.UpdateBackupPolicy(scheduleId).BackupScheduleSpec(*backupScheduleSpec).Execute()
_, r, err = authApi.UpdateBackupPolicy(clusterId, scheduleId).ScheduleSpecV2(backupScheduleSpec).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
Expand Down Expand Up @@ -333,7 +310,6 @@ func generateCronExpression(daysOfWeek string, backupTime string) string {
if found {
cronDays = append(cronDays, cronDay)
}

}

cronExpr := fmt.Sprintf("%d %d * * %s", minute, hour, strings.Join(cronDays, ","))
Expand Down Expand Up @@ -361,6 +337,9 @@ func init() {
updatePolicyCmd.Flags().Int32("retention-period-in-days", 1, "[REQUIRED] Retention period of the backup in days.")
updatePolicyCmd.MarkFlagRequired("retention-period-in-days")
updatePolicyCmd.Flags().Int32("full-backup-frequency-in-days", 1, "[OPTIONAL] Frequency of full backup in days.")
if util.IsFeatureFlagEnabled(util.INCREMENTAL_BACKUP) {
updatePolicyCmd.Flags().Int32("incremental-backup-frequency-in-minutes", 60, "[OPTIONAL] Frequency of incremental backup in minutes.")
}
updatePolicyCmd.Flags().String("full-backup-schedule-days-of-week", "", "[OPTIONAL] Days of the week when the backup has to run. A comma separated list of the first two letters of the days of the week. Eg: 'Mo,Tu,Sa'")
updatePolicyCmd.Flags().String("full-backup-schedule-time", "", "[OPTIONAL] Time of the day at which the backup has to run. Please specify local time in 24 hr HH:MM format. Eg: 15:04")
updatePolicyCmd.MarkFlagsRequiredTogether("full-backup-schedule-days-of-week", "full-backup-schedule-time")
Expand Down
16 changes: 8 additions & 8 deletions cmd/backup_shedule_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ var _ = Describe("BackupSchedules", func() {
args []string
responseAccount openapi.AccountListResponse
responseProject openapi.AccountListResponse
responseListBackupSchedules openapi.ScheduleListResponse
responseListCronBackupSchedules openapi.ScheduleListResponse
responseListBackupSchedules openapi.ScheduleListResponseV2
responseListCronBackupSchedules openapi.ScheduleListResponseV2
responseListClusters openapi.ClusterListResponse
)

Expand Down Expand Up @@ -74,7 +74,7 @@ var _ = Describe("BackupSchedules", func() {
It("should return list of backup schedules with a paused schedule", func() {
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/backup-schedules"),
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/clusters/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/backup-schedules"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseListBackupSchedules),
),
)
Expand All @@ -83,16 +83,16 @@ var _ = Describe("BackupSchedules", func() {
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
o := string(session.Out.Contents()[:])
expected := `Time Interval(days) Days of the Week Backup Start Time Retention Period(days) State
1 NA NA 8 PAUSED` + "\n"
expected := `Time Interval(days) Incremental Time Interval(minutes) Days of the Week Backup Start Time Retention Period(days) State
1 60 NA NA 8 PAUSED` + "\n"
Expect(o).Should(Equal(expected))

session.Kill()
})
It("should return backup schedules with cron expression with an active schedule", func() {
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/backup-schedules"),
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/clusters/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/backup-schedules"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseListCronBackupSchedules),
),
)
Expand All @@ -103,8 +103,8 @@ var _ = Describe("BackupSchedules", func() {
o := string(session.Out.Contents()[:])

fmt.Println(o)
expected := `Time Interval(days) Days of the Week Backup Start Time Retention Period(days) State
NA Su,We,Fr ` + getLocalTime("2 3 * * *") + ` 8 ACTIVE` + "\n"
expected := `Time Interval(days) Incremental Time Interval(minutes) Days of the Week Backup Start Time Retention Period(days) State
NA NA Su,We,Fr ` + getLocalTime("2 3 * * *") + ` 8 ACTIVE` + "\n"
Expect(o).Should(Equal(expected))
fmt.Println(expected)

Expand Down
16 changes: 6 additions & 10 deletions cmd/test/fixtures/list-backup-schedules-cron.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,19 @@
"data": [
{
"spec": {
"description": "Default backup schedule for fabulous-bobolink",
"time_interval_in_days": 0,
"cron_expression": "2 3 * * 0,3,5",
"retention_period_in_days": 8,
"state": "ACTIVE"
},
"info": {
"id": "c55f06da-b6ea-44c6-a37c-b8a76d6922e4",
"task_type": "CREATE_BACKUP",
"task_params": {
"cluster_id": "5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8",
"retention_period_in_days": 8,
"description": "Default backup schedule for polite-haddock"
},
"entity_type": "CLUSTER",
"entity_id": "5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8",
"schedule_type": "SYSTEM_DEFAULT",
"schedule_type": "USER_DEFINED",
"last_schedule_time": null,
"next_schedule_time": null
"next_schedule_time": "2024-01-25T23:16Z",
"last_incremental_schedule_time": null,
"next_incremental_schedule_time": null
}
}
]
Expand Down
17 changes: 7 additions & 10 deletions cmd/test/fixtures/list-backup-schedules.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,20 @@
"data": [
{
"spec": {
"description": "Default backup schedule for fabulous-bobolink",
"time_interval_in_days": 1,
"cron_expression": null,
"retention_period_in_days": 8,
"incremental_interval_in_minutes": 60,
"state": "PAUSED"
},
"info": {
"id": "c55f06da-b6ea-44c6-a37c-b8a76d6922e4",
"task_type": "CREATE_BACKUP",
"task_params": {
"cluster_id": "5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8",
"retention_period_in_days": 8,
"description": "Default backup schedule for polite-haddock"
},
"entity_type": "CLUSTER",
"entity_id": "5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8",
"schedule_type": "SYSTEM_DEFAULT",
"schedule_type": "USER_DEFINED",
"last_schedule_time": null,
"next_schedule_time": null
"next_schedule_time": "2024-01-25T23:16Z",
"last_incremental_schedule_time": null,
"next_incremental_schedule_time": null
}
}
]
Expand Down
1 change: 1 addition & 0 deletions cmd/util/feature_flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const (
TOOLS FeatureFlag = "TOOLS"
AZURE_CIDR_ALLOWED FeatureFlag = "AZURE_CIDR_ALLOWED"
ENTERPRISE_SECURITY FeatureFlag = "ENTERPRISE_SECURITY"
INCREMENTAL_BACKUP FeatureFlag = "INCREMENTAL_BACKUP"
)

func (f FeatureFlag) String() string {
Expand Down
20 changes: 10 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ require (
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
github.com/jayco/go-emoji-flag v0.0.0-20190810054606-01604da018da
github.com/mattn/go-runewidth v0.0.14
github.com/onsi/ginkgo/v2 v2.13.0
github.com/onsi/gomega v1.30.0
github.com/onsi/ginkgo/v2 v2.15.0
github.com/onsi/gomega v1.31.1
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/pkg/errors v0.9.1
github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.17.0
github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20231201193810-43fdeee78dc9
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20240123191212-6c0c5862cc02
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/mod v0.12.0
golang.org/x/term v0.13.0
golang.org/x/mod v0.14.0
golang.org/x/term v0.15.0
gotest.tools/v3 v3.4.0
)

Expand All @@ -46,7 +46,7 @@ require (
github.com/cloudflare/circl v1.3.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
Expand Down Expand Up @@ -74,11 +74,11 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
Loading

0 comments on commit 7f2f9c5

Please sign in to comment.