Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bulk PITR config creation #285

Merged
merged 1 commit into from
Dec 20, 2024
Merged

Bulk PITR config creation #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 88 additions & 24 deletions cmd/cluster/pitr-config/pitr_config_ops.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ package pitrconfig

import (
"fmt"
"math"
"os"
"strconv"
"strings"

"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
Expand All @@ -29,6 +32,7 @@ import (
)

var ClusterName string
var allPitrConfigSpecs []string

var listPitrConfigCmd = &cobra.Command{
Use: "list",
Expand Down Expand Up @@ -116,47 +120,113 @@ var createPitrConfigCmd = &cobra.Command{
logrus.Fatal(err)
}

namespaceName, _ := cmd.Flags().GetString("namespace-name")
namespaceType, _ := cmd.Flags().GetString("namespace-type")
validateNamespaceNameType(namespaceName, namespaceType)
retentionPeriod, _ := cmd.Flags().GetInt32("retention-period-in-days")
pitrConfigSpecs, err := ParsePitrConfigSpecs(authApi, allPitrConfigSpecs)
if err != nil {
logrus.Fatalf("Error while parsing PITR Config specs: %s", ybmAuthClient.GetApiErrorDetails(err))
return
}

pitrConfigSpec, err := authApi.CreatePitrConfigSpec(namespaceName, namespaceType, retentionPeriod)
bulkPitrConfigSpec, err := authApi.CreateBulkPitrConfigSpec(pitrConfigSpecs)
if err != nil {
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}

resp, r, err := authApi.CreatePitrConfig(clusterID).DatabasePitrConfigSpec(*pitrConfigSpec).Execute()
resp, r, err := authApi.CreatePitrConfig(clusterID).BulkCreateDatabasePitrConfigSpec(*bulkPitrConfigSpec).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}
pitrConfigId := resp.Data.Info.Id
pitrConfigsData := resp.GetData()

msg := fmt.Sprintf("The PITR Configuration for %s namespace %s in cluster %s is being created\n\n", namespaceType, formatter.Colorize(namespaceName, formatter.GREEN_COLOR), formatter.Colorize(ClusterName, formatter.GREEN_COLOR))
msg := fmt.Sprintf("The requested PITR Configurations are being created\n\n")

if viper.GetBool("wait") {
handleTaskCompletion(authApi, clusterID, msg, ybmclient.TASKTYPEENUM_ENABLE_DB_PITR)
fmt.Printf("Successfully created PITR configuration.\n\n")

getConfigResp, r, err := authApi.GetPitrConfig(clusterID, *pitrConfigId).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
handleTaskCompletion(authApi, clusterID, msg, ybmclient.TASKTYPEENUM_BULK_ENABLE_DB_PITR)
fmt.Printf("Successfully created PITR configurations.\n\n")
createdConfigsData := []ybmclient.DatabasePitrConfigData{}
for _, configData := range pitrConfigsData {
configId := configData.Info.Id
getConfigResp, r, err := authApi.GetPitrConfig(clusterID, *configId).Execute()
if err != nil {
logrus.Debugf("Full HTTP response: %v", r)
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}
createdConfigsData = append(createdConfigsData, getConfigResp.GetData())
}
pitrConfigData := getConfigResp.GetData()

pitrConfigCtx := formatter.Context{
Output: os.Stdout,
Format: formatter.NewPitrConfigFormat(viper.GetString("output")),
}

formatter.SinglePitrConfigWrite(pitrConfigCtx, pitrConfigData)
formatter.PitrConfigWrite(pitrConfigCtx, createdConfigsData)
} else {
fmt.Println(msg)
}
},
}

// Parse array of PITR config spec string to params
func ParsePitrConfigSpecs(authApi *ybmAuthClient.AuthApiClient, configSpecs []string) ([]ybmclient.DatabasePitrConfigSpec, error) {
var err error
pitrConfigSpecs := []ybmclient.DatabasePitrConfigSpec{}

for _, configSpec := range configSpecs {
var namespaceNameProvided bool
var namespaceTypeProvided bool
var retentionPeriodProvided bool
spec := *ybmclient.NewDatabasePitrConfigSpecWithDefaults()
configSpec := strings.ReplaceAll(configSpec, " ", "")

for _, subSpec := range strings.Split(configSpec, ",") {
if !strings.Contains(subSpec, "=") {
return nil, fmt.Errorf("namespace-name, namespace-type and retention-period-in-days must be provided as key value pairs for each PITR Config to be created")
}
kvp := strings.Split(subSpec, "=")
key := kvp[0]
val := kvp[1]
n := 0
err = nil
switch key {
case "namespace-name":
if len(val) == 0 {
return nil, fmt.Errorf("Namespace name must be provided.")
}
spec.SetDatabaseName(val)
namespaceNameProvided = true
case "namespace-type":
if !(val == "YCQL" || val == "YSQL") {
return nil, fmt.Errorf("Only YCQL or YSQL namespace types are allowed.")
}
spec.SetDatabaseType(ybmclient.YbApiEnum(val))
namespaceTypeProvided = true
case "retention-period-in-days":
n, err = strconv.Atoi(val)
if err != nil {
return nil, err
}
if n > 1 && n <= math.MaxInt32 {
retentionPeriod := int32(n)
spec.SetRetentionPeriod(retentionPeriod)
retentionPeriodProvided = true
} else {
return nil, fmt.Errorf("Minimum retention period is 2 days")
}
}
}
if !(namespaceNameProvided && namespaceTypeProvided && retentionPeriodProvided) {
return nil, fmt.Errorf("namespace-name, namespace-type and retention-period-in-days must be provided for each PITR Config to be created")
}
pitrConfigSpecs = append(pitrConfigSpecs, spec)
}

if len(pitrConfigSpecs) == 0 {
return nil, fmt.Errorf("namespace-name, namespace-type and retention-period-in-days must be provided for each PITR Config to be created")
}

return pitrConfigSpecs, nil
}

var restorePitrConfigCmd = &cobra.Command{
Use: "restore",
Short: "Restore namespace via PITR Config for a cluster",
Expand Down Expand Up @@ -309,13 +379,7 @@ func init() {
describePitrConfigCmd.MarkFlagRequired("namespace-type")

util.AddCommandIfFeatureFlag(PitrConfigCmd, createPitrConfigCmd, util.PITR_CONFIG)
createPitrConfigCmd.Flags().SortFlags = false
createPitrConfigCmd.Flags().String("namespace-name", "", "[REQUIRED] Namespace for which the PITR Config is to be created.")
createPitrConfigCmd.MarkFlagRequired("namespace-name")
createPitrConfigCmd.Flags().String("namespace-type", "", "[REQUIRED] The type of the namespace. Available options are YCQL and YSQL")
createPitrConfigCmd.MarkFlagRequired("namespace-type")
createPitrConfigCmd.Flags().Int32("retention-period-in-days", 2, "[REQUIRED] The time duration in days to retain a snapshot for.")
createPitrConfigCmd.MarkFlagRequired("retention-period-in-days")
createPitrConfigCmd.Flags().StringArrayVarP(&allPitrConfigSpecs, "pitr-config", "p", []string{}, `[REQUIRED] Information for the PITR Configs to be created. All values are mandatory. Available options for namespace type are YCQL and YSQL. Minimum retention period is 2 days. Please provide key value pairs namespace-name=<namespace-name>,namespace-type=<namespace-type>,retention-period-in-days=<retention-period-in-days>.`)

util.AddCommandIfFeatureFlag(PitrConfigCmd, restorePitrConfigCmd, util.PITR_RESTORE)
restorePitrConfigCmd.Flags().SortFlags = false
Expand Down
53 changes: 49 additions & 4 deletions cmd/pitr_config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ var _ = Describe("PITR Configs Test", func() {
responseListCluster openapi.ClusterListResponse
responseListPITRConfig openapi.ClusterPitrConfigListResponse
responseGetPITRConfig openapi.DatabasePitrConfigResponse
responseCreatePITRConfig openapi.CreateDatabasePitrConfigResponse
responseCreatePITRConfig openapi.BulkCreateDatabasePitrConfigResponse
responseRestoreViaPITRConfig openapi.RestoreDatabaseViaPitrResponse
)

Expand Down Expand Up @@ -97,22 +97,67 @@ test_ysql_db YSQL 5 QUEUED 654321
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseCreatePITRConfig),
),
)
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--namespace-name", "test_ysql_db", "--namespace-type", "YSQL", "--retention-period-in-days", "5")
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=test_ysql_db, namespace-type=YSQL, retention-period-in-days=5", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`The PITR Configuration for YSQL namespace test_ysql_db in cluster stunning-sole is being created`))
Expect(session.Out).Should(gbytes.Say(`The requested PITR Configurations are being created`))
session.Kill()
})

It("Should fail if invalid namespace type in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--namespace-name", "test_ysql_db", "--namespace-type", "PGSQL", "--retention-period-in-days", "5")
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=test_ysql_db, namespace-type=PGSQL, retention-period-in-days=5")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("Only YCQL or YSQL namespace types are allowed."))
session.Kill()
})

It("Should fail if empty namespace name in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=, namespace-type=YSQL, retention-period-in-days=5", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("Namespace name must be provided."))
session.Kill()
})

It("Should fail if invalid key value pairs in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name test_ysql_db, namespace-type=YSQL, retention-period-in-days=5", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("namespace-name, namespace-type and retention-period-in-days must be provided as key value pairs for each PITR Config to be created"))
session.Kill()
})

It("Should fail if all required params are not in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=test_ysql_db, namespace-type=YSQL ", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("namespace-name, namespace-type and retention-period-in-days must be provided for each PITR Config to be created"))
session.Kill()
})

It("Should fail if non int retention period in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=test_ysql_db, namespace-type=YSQL, retention-period-in-days=five", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("invalid syntax"))
session.Kill()
})

It("Should fail if less than one day retention period in PITR Config", func() {
cmd := exec.Command(compiledCLIPath, "cluster", "pitr-config", "create", "--cluster-name", "stunning-sole", "--pitr-config", "namespace-name=test_ysql_db, namespace-type=YSQL, retention-period-in-days=1", "--pitr-config", "namespace-name=test_ycql_db, namespace-type=YCQL, retention-period-in-days=3")
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Err).Should(gbytes.Say("Minimum retention period is 2 days"))
session.Kill()
})
})

var _ = Describe("Restore cluster namespace via PITR config", func() {
Expand Down
47 changes: 33 additions & 14 deletions cmd/test/fixtures/create-cluster-pitr-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,38 @@
{
"data": {
"spec": {
"database_type": "YSQL",
"database_name": "test_ysql_db",
"retention_period": 5
"data": [
{
"spec": {
"database_type": "YSQL",
"database_name": "test_ysql_db",
"retention_period": 5
},
"info": {
"id": "3fb11555-b561-4e8f-b0ed-e77b8374cbc3",
"cluster_id": "95c522b3-dfd8-4654-ab10-99f59179ad7c",
"metadata": {
"created_on": "2024-08-07T16:26:08.435Z",
"updated_on": "2024-08-07T16:26:08.435Z"
},
"backup_interval": 86400,
"state": "QUEUED"
}
},
"info": {
"id": "3fb11555-b561-4e8f-b0ed-e77b8374cbc3",
"cluster_id": "95c522b3-dfd8-4654-ab10-99f59179ad7c",
"metadata": {
"created_on": "2024-08-07T16:26:08.435Z",
"updated_on": "2024-08-07T16:26:08.435Z"
{
"spec": {
"database_type": "YCQL",
"database_name": "test_ycql_db",
"retention_period": 3
},
"backup_interval": 86400,
"state": "QUEUED"
"info": {
"id": "249f9bf1-4276-4c60-8ab3-2bf1b2f6f1aa",
"cluster_id": "95c522b3-dfd8-4654-ab10-99f59179ad7c",
"metadata": {
"created_on": "2024-08-03T11:38:10.838Z",
"updated_on": "2024-08-03T11:38:10.838Z"
},
"backup_interval": 86400,
"state": "QUEUED"
}
}
}
]
}
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.17.0
github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241129094603-513ccfc1e5ae
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241219183048-50fe86c058d8
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/mod v0.20.0
golang.org/x/term v0.25.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,8 +282,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 h1:J6v8awz+me+xeb/cUTotKgceAYouhIB3pjzgRd6IlGk=
github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816/go.mod h1:tzym/CEb5jnFI+Q0k4Qq3+LvRF4gO3E2pxS8fHP8jcA=
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241129094603-513ccfc1e5ae h1:DPZFx2PSJhrCfoZ5vjre2rwvoltlBB75chAlL0Zdapc=
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241129094603-513ccfc1e5ae/go.mod h1:5vW0xIzIZw+1djkiWKx0qqNmqbRBSf4mjc4qw8lIMik=
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241219183048-50fe86c058d8 h1:criIjOTBfBOy0cZ23Qh2sOI0KLL3P4WKutnUhovUHPA=
github.com/yugabyte/yugabytedb-managed-go-client-internal v0.0.0-20241219183048-50fe86c058d8/go.mod h1:5vW0xIzIZw+1djkiWKx0qqNmqbRBSf4mjc4qw8lIMik=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
7 changes: 4 additions & 3 deletions internal/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1549,9 +1549,10 @@ func (a *AuthApiClient) ListClusterPitrConfigs(clusterId string) ybmclient.ApiLi
return a.ApiClient.ClusterApi.ListClusterPitrConfigs(a.ctx, a.AccountID, a.ProjectID, clusterId)
}

func (a *AuthApiClient) CreatePitrConfigSpec(databaseName string, databaseType string, retentionPeriodInDays int32) (*ybmclient.DatabasePitrConfigSpec, error) {
pitrConfigSpec := ybmclient.NewDatabasePitrConfigSpec(ybmclient.YbApiEnum(databaseType), databaseName, retentionPeriodInDays)
return pitrConfigSpec, nil
func (a *AuthApiClient) CreateBulkPitrConfigSpec(specs []ybmclient.DatabasePitrConfigSpec) (*ybmclient.BulkCreateDatabasePitrConfigSpec, error) {
bulkPitrConfigSpec := ybmclient.NewBulkCreateDatabasePitrConfigSpec()
bulkPitrConfigSpec.SetPitrConfigSpecs(specs)
return bulkPitrConfigSpec, nil
}

func (a *AuthApiClient) CreatePitrConfig(clusterId string) ybmclient.ApiCreateDatabasePitrConfigRequest {
Expand Down
Loading