From a2d3dd58f2aac103d3475b0ca057981253e6a22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Duy=20Ph=C6=B0=C6=A1ng?= Date: Fri, 8 Sep 2023 23:49:02 +0700 Subject: [PATCH] support cloudsql and rds (#33) * init support for cloudsql * update cloudsql package * update cloudsql package * skip replica instance * init support for rds instances * pause rds custom automation * update docs --- .gitignore | 1 + README.md | 9 ++ docker-compose.yml | 30 ++-- go.mod | 10 +- go.sum | 15 +- internal/api/api.go | 6 +- internal/api/handler/change_state_v1alpha1.go | 54 ++++--- internal/constants/const.go | 2 + internal/worker/worker.go | 2 +- pkg/api/service/v1alpha1/service.go | 4 +- pkg/aws/client.go | 2 +- pkg/aws/ec2/client.go | 8 +- pkg/aws/rds/client.go | 151 ++++++++++++++++++ pkg/aws/rds/client_test.go | 22 +++ pkg/gcp/client.go | 17 +- pkg/gcp/cloudsql/client.go | 88 ++++++++++ pkg/gcp/cloudsql/client_test.go | 38 +++++ pkg/gcp/gce/client.go | 12 +- pkg/gcp/utils/filter.go | 11 ++ 19 files changed, 416 insertions(+), 66 deletions(-) create mode 100644 pkg/aws/rds/client.go create mode 100644 pkg/aws/rds/client_test.go create mode 100644 pkg/gcp/cloudsql/client.go create mode 100644 pkg/gcp/cloudsql/client_test.go create mode 100644 pkg/gcp/utils/filter.go diff --git a/.gitignore b/.gitignore index 6f406f8..e2f064d 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,4 @@ app volume .env local-env.sh +local.sh diff --git a/README.md b/README.md index cfa388e..185496d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,15 @@ Scheduler for compute instances across clouds. A Golang port of [Doiintl's Zorya [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://raw.githubusercontent.com/nduyphuong/gorya/dev/LICENSE) [![Build status](https://github.com/nduyphuong/gorya/actions/workflows/release.yml/badge.svg)](https://github.com/nduyphuong/gorya/actions) +# Support Resources +- AWS: + - [X] EC2 + - [X] RDS + - [ ] EKS +- GCP: + - [X] EC2 + - [X] CLOUDSQL + - [ ] GKE ## Building Gorya ### Software requirements diff --git a/docker-compose.yml b/docker-compose.yml index 83b4cab..2b5c3c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,21 +57,21 @@ services: networks: - gorya - gorya-ui: -# image: ghcr.io/nduyphuong/gorya-ui - container_name: gorya-ui - ports: - - "3000:3000" - build: - context: . - dockerfile: frontend.Dockerfile - environment: - NODE_OPTIONS: --openssl-legacy-provider - GORYA_API_ADDR: http://gorya-backend:9000 - depends_on: - - gorya-backend - networks: - - gorya +# gorya-ui: +# # image: ghcr.io/nduyphuong/gorya-ui +# container_name: gorya-ui +# ports: +# - "3000:3000" +# build: +# context: . +# dockerfile: frontend.Dockerfile +# environment: +# NODE_OPTIONS: --openssl-legacy-provider +# GORYA_API_ADDR: http://gorya-backend:9000 +# depends_on: +# - gorya-backend +# networks: +# - gorya keycloak: image: koolwithk/keycloak-arm:16.0.0 diff --git a/go.mod b/go.mod index 4a3bffb..a02e1b7 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/nduyphuong/gorya go 1.20 require ( - github.com/aws/aws-sdk-go-v2 v1.20.3 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.35 github.com/aws/aws-sdk-go-v2/credentials v1.13.34 github.com/aws/aws-sdk-go-v2/service/ec2 v1.113.1 + github.com/aws/aws-sdk-go-v2/service/rds v1.54.0 github.com/aws/aws-sdk-go-v2/service/sts v1.21.4 github.com/coreos/go-oidc v2.2.1+incompatible github.com/go-redis/redis/v8 v8.11.5 @@ -19,6 +20,7 @@ require ( github.com/stretchr/testify v1.8.4 golang.org/x/net v0.14.0 golang.org/x/oauth2 v0.11.0 + golang.org/x/sync v0.2.0 google.golang.org/api v0.126.0 gorm.io/datatypes v1.2.0 gorm.io/driver/mysql v1.5.1 @@ -30,10 +32,10 @@ require ( cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.41 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.4 // indirect github.com/aws/smithy-go v1.14.2 // indirect diff --git a/go.sum b/go.sum index c2a31bf..a0c6231 100644 --- a/go.sum +++ b/go.sum @@ -6,24 +6,30 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/aws-sdk-go-v2 v1.20.3 h1:lgeKmAZhlj1JqN43bogrM75spIvYnRxqTAh1iupu1yE= github.com/aws/aws-sdk-go-v2 v1.20.3/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/config v1.18.35 h1:uU9rgCzrW/pVRUUlRULiwKQe8RoEDst1NQu4Qo8kOtk= github.com/aws/aws-sdk-go-v2/config v1.18.35/go.mod h1:7xF1yr9GBMfYRQI4PLHO8iceqKLM6DpGVEvXI38HB/A= github.com/aws/aws-sdk-go-v2/credentials v1.13.34 h1:/EYG4lzayDd5PY6HQQ2Qyj/cD6CR3kz96BjTZAO5tNo= github.com/aws/aws-sdk-go-v2/credentials v1.13.34/go.mod h1:+wgdxCGNulHme6kTMZuDL9KOagLPloemoYkfjpQkSEU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10 h1:mgOrtwYfJZ4e3QJe1TrliC/xIkauafGMdLLuCExOqcs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10/go.mod h1:wMsSLVM2hRpDVhd+3dtLUzqwm7/fjuhNN+b1aOLDt6g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40 h1:CXceCS9BrDInRc74GDCQ8Qyk/Gp9VLdK+Rlve+zELSE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40/go.mod h1:5kKmFhLeOVy6pwPDpDNA6/hK/d6URC98pqDDqHgdBx4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34 h1:B+nZtd22cbko5+793hg7LEaTeLMiZwlgCLUrN5Y0uzg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34/go.mod h1:RZP0scceAyhMIQ9JvFp7HvkpcgqjL4l/4C+7RAeGbuM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.41 h1:EcSFdpLdkF3FWizimox0qYLuorn9e4PNMR27mvshGLs= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.41/go.mod h1:mKxUXW+TuwpCKKHVlmHGVVuBi9y9LKW8AiQodg23M5E= github.com/aws/aws-sdk-go-v2/service/ec2 v1.113.1 h1:2wyKWQM+5+lMaSNU9RCwIVNRYJZjiXdNUJfavh5hCTM= github.com/aws/aws-sdk-go-v2/service/ec2 v1.113.1/go.mod h1:YBN5ov75u3UBgWKzV9ZlXu+Jb9oLoA2MqrAVJjaHGLc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34 h1:JwvXk+1ePAD9xkFHprhHYqwsxLDcbNFsPI1IAT2sPS0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34/go.mod h1:ytsF+t+FApY2lFnN51fJKPhH6ICKOPXKEcwwgmJEdWI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/rds v1.54.0 h1:FmExQnV6PXPAwP2DT3nXlWyKtCJ30gCEQIu4MUOuESo= +github.com/aws/aws-sdk-go-v2/service/rds v1.54.0/go.mod h1:UNv1vk1fU1NJefzteykVpVLA88w4WxB05g3vp2kQhYM= github.com/aws/aws-sdk-go-v2/service/sso v1.13.4 h1:WZPZ7Zf6Yo13lsfTetFrLU/7hZ9CXESDpdIHvmLxQFQ= github.com/aws/aws-sdk-go-v2/service/sso v1.13.4/go.mod h1:FP05hDXTLouXwAMQ1swqybHy7tHySblMkBMKSumaKg0= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.4 h1:pYFM2U/3/4RLrlMSYXwL1XPBCWvaePk2p+0+i/BgHOs= @@ -217,6 +223,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/api/api.go b/internal/api/api.go index e7bd6af..08bd941 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -71,6 +71,7 @@ func (s *server) Serve(ctx context.Context, l net.Listener) error { providers := strings.Split(os.GetEnv("GORYA_ENABLED_PROVIDERS", ""), ",") ticker := time.NewTicker(30 * time.Second) for _, provider := range providers { + provider = strings.ToLower(provider) if provider != constants.PROVIDER_AWS && provider != constants.PROVIDER_GCP && provider != constants.PROVIDER_AZURE { continue } @@ -89,7 +90,7 @@ func (s *server) Serve(ctx context.Context, l net.Listener) error { } } } - //c.credentialRef["arn:aws:iam::043159268388:role/test"] = true + // c.credentialRef["arn:aws:iam::043159268388:role/test"] = true s.aws, err = aws.NewPool( ctx, c.credentialRef, @@ -132,6 +133,7 @@ func (s *server) Serve(ctx context.Context, l net.Listener) error { } } } + c.credentialRef["priv-sa@target-project-397310.iam.gserviceaccount.com"] = true s.gcp, err = gcp.NewPool( ctx, c.credentialRef, @@ -166,7 +168,7 @@ func (s *server) Serve(ctx context.Context, l net.Listener) error { PopInterval: 2 * time.Second, }, }) - path, svcHandler := svcv1alpha1.NewGoryaServiceHandler(ctx, s.sc, s) + path, svcHandler := svcv1alpha1.NewGoryaServiceHandler(ctx, s) mux.Handle(path, svcHandler) srv := &http.Server{ Handler: h2c.NewHandler(mux, &http2.Server{}), diff --git a/internal/api/handler/change_state_v1alpha1.go b/internal/api/handler/change_state_v1alpha1.go index f0e9586..9328abe 100644 --- a/internal/api/handler/change_state_v1alpha1.go +++ b/internal/api/handler/change_state_v1alpha1.go @@ -32,30 +32,38 @@ func ChangeStateV1alpha1(ctx context.Context, awsClientPool *aws.ClientPool, gcp http.Error(w, pkgerrors.Wrap(err, "decode state change request body").Error(), http.StatusBadRequest) return } - if awsClientPool != nil && m.Provider == constants.PROVIDER_AWS { - awsClient, ok := awsClientPool.GetForCredential(m.CredentialRef) - if !ok { - http.Error(w, fmt.Errorf("client not found for credential %v", m.CredentialRef).Error(), - http.StatusBadRequest) - return + switch m.Provider { + case constants.PROVIDER_AWS: + if awsClientPool != nil { + awsClient, ok := awsClientPool.GetForCredential(m.CredentialRef) + if !ok { + http.Error(w, fmt.Errorf("client not found for credential %v", m.CredentialRef).Error(), + http.StatusBadRequest) + return + } + compute := awsClient.EC2() + if err := compute.ChangeStatus(ctx, m.Action, m.TagKey, m.TagValue); err != nil { + http.Error(w, pkgerrors.Wrap(err, "change compute status").Error(), http.StatusInternalServerError) + return + } } - compute := awsClient.EC2() - if err := compute.ChangeStatus(ctx, m.Action, m.TagKey, m.TagValue); err != nil { - http.Error(w, pkgerrors.Wrap(err, "change compute status").Error(), http.StatusInternalServerError) - return - } - } - if gcpClientPool != nil && m.Provider == constants.PROVIDER_GCP { - gcpClient, ok := gcpClientPool.GetForCredential(m.CredentialRef) - if !ok { - http.Error(w, fmt.Errorf("client not found for credential %v", m.CredentialRef).Error(), - http.StatusBadRequest) - return - } - compute := gcpClient.GCE() - if err := compute.ChangeStatus(ctx, m.Action, m.TagKey, m.TagValue); err != nil { - http.Error(w, pkgerrors.Wrap(err, "change compute status").Error(), http.StatusInternalServerError) - return + case constants.PROVIDER_GCP: + if gcpClientPool != nil { + gcpClient, ok := gcpClientPool.GetForCredential(m.CredentialRef) + if !ok { + http.Error(w, fmt.Errorf("client not found for credential %v", m.CredentialRef).Error(), + http.StatusBadRequest) + return + } + compute := gcpClient.GCE() + if err := compute.ChangeStatus(ctx, m.Action, m.TagKey, m.TagValue); err != nil { + http.Error(w, pkgerrors.Wrap(err, "change compute status").Error(), http.StatusInternalServerError) + } + cloudSql := gcpClient.CloudSQL() + if err := cloudSql.ChangeStatus(ctx, m.Action, m.TagKey, m.TagValue); err != nil { + http.Error(w, pkgerrors.Wrap(err, "change sql instances status").Error(), http.StatusInternalServerError) + return + } } } } diff --git a/internal/constants/const.go b/internal/constants/const.go index 6580cd5..37b74f3 100644 --- a/internal/constants/const.go +++ b/internal/constants/const.go @@ -1,6 +1,8 @@ package constants const ( + OffStatus int = iota + OnStatus PROVIDER_AWS = "aws" PROVIDER_GCP = "gcp" PROVIDER_AZURE = "azure" diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 52fe007..e2135ac 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -74,7 +74,7 @@ func (c *client) Process(ctx context.Context, stop <-chan struct{}, errChan chan fmt.Println("stop background process") return case task := <-resultChan: - fmt.Printf("popped item %v", task) + fmt.Printf("popped item %v \n", task) var elem QueueElem err := json.Unmarshal([]byte(task), &elem) if err != nil { diff --git a/pkg/api/service/v1alpha1/service.go b/pkg/api/service/v1alpha1/service.go index 41aeee6..20686fe 100644 --- a/pkg/api/service/v1alpha1/service.go +++ b/pkg/api/service/v1alpha1/service.go @@ -5,8 +5,6 @@ import ( "net/http" "github.com/nduyphuong/gorya/internal/api/middleware" - - "github.com/nduyphuong/gorya/internal/store" ) //go:generate mockery --name GoryaServiceHandler @@ -43,7 +41,7 @@ const ( // NewGoryaServiceHandler builds an HTTP handler from the service implementation. It returns the // // path on which to mount the handler and the handler itself. -func NewGoryaServiceHandler(ctx context.Context, store store.Interface, svc GoryaServiceHandler) (string, +func NewGoryaServiceHandler(ctx context.Context, svc GoryaServiceHandler) (string, http.Handler) { mux := http.NewServeMux() mux.Handle(GoryaGetTimeZoneProcedure, middleware.JWTAuthorization(svc.GetTimeZone(), "get-timezone", ctx)) diff --git a/pkg/aws/client.go b/pkg/aws/client.go index 474ea6b..c298184 100644 --- a/pkg/aws/client.go +++ b/pkg/aws/client.go @@ -61,7 +61,7 @@ func (b *ClientPool) GetForCredential(name string) (Interface, bool) { if !ok { return nil, false } - fmt.Printf("got client from pool for %s", name) + fmt.Printf("got client from pool for %s\n", name) return i, true } diff --git a/pkg/aws/ec2/client.go b/pkg/aws/ec2/client.go index 410b4c7..2353833 100644 --- a/pkg/aws/ec2/client.go +++ b/pkg/aws/ec2/client.go @@ -15,17 +15,17 @@ type Interface interface { ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) } -type Client struct { +type client struct { ec2 *ec2.Client } -func NewFromConfig(cfg aws.Config) (*Client, error) { - c := &Client{} +func NewFromConfig(cfg aws.Config) (*client, error) { + c := &client{} c.ec2 = ec2.NewFromConfig(cfg) return c, nil } -func (c *Client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) { +func (c *client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) { if to != 0 && to != 1 { return errors.New("to must have value of 0 or 1") } diff --git a/pkg/aws/rds/client.go b/pkg/aws/rds/client.go new file mode 100644 index 0000000..d6bfe07 --- /dev/null +++ b/pkg/aws/rds/client.go @@ -0,0 +1,151 @@ +package rds + +import ( + "context" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + pkgerrors "github.com/pkg/errors" + + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/nduyphuong/gorya/internal/constants" + "github.com/nduyphuong/gorya/internal/logging" +) + +var ErrInvalidResourceStatus = errors.New("invalid resource status") + +type Interface interface { + ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) +} + +type client struct { + rds *rds.Client +} + +func NewFromConfig(cfg aws.Config) (*client, error) { + c := &client{} + c.rds = rds.NewFromConfig(cfg) + return c, nil +} + +func (c *client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) { + logger := logging.LoggerFromContext(ctx) + if to != constants.OffStatus && to != constants.OnStatus { + return ErrInvalidResourceStatus + } + dbClusters, err := c.describeDBCluster(ctx) + if err != nil { + return pkgerrors.Wrap(err, "describe db clusters") + } + // fmt.Printf("dbClusters: %v\n", dbClusters) + dbInstances, err := c.describeDBInstance(ctx) + if err != nil { + return pkgerrors.Wrap(err, "describe db instances") + } + // fmt.Printf("dbInstances: %v\n", dbInstances) + instanceToClusterMap := map[*string]*string{} + for _, cluster := range dbClusters { + // fmt.Printf("cluster.DBClusterIdentifier: %v\n", *cluster.DBClusterIdentifier) + for _, tag := range cluster.TagList { + // fmt.Printf("tag.Key: %v\n", *tag.Key) + // fmt.Printf("tag.Value: %v\n", *tag.Value) + if *tag.Key != tagKey || *tag.Value != tagValue { + continue + } + switch to { + case constants.OnStatus: + _, err = c.rds.StartDBCluster(ctx, &rds.StartDBClusterInput{ + DBClusterIdentifier: cluster.DBClusterIdentifier, + }) + if err != nil { + logger.Error(pkgerrors.Wrap(err, fmt.Sprintf("start db cluster %s", *cluster.DBClusterIdentifier))) + } + case constants.OffStatus: + _, err = c.rds.StopDBCluster(ctx, &rds.StopDBClusterInput{ + DBClusterIdentifier: cluster.DBClusterIdentifier, + }) + if err != nil { + logger.Error(pkgerrors.Wrap(err, fmt.Sprintf("stop db cluster %s", *cluster.DBClusterIdentifier))) + } + } + + } + for _, member := range cluster.DBClusterMembers { + instanceToClusterMap[member.DBInstanceIdentifier] = cluster.DBClusterIdentifier + } + } + for _, instance := range dbInstances { + for _, tag := range instance.TagList { + if *tag.Key != tagKey || *tag.Value != tagValue || instanceToClusterMap[instance.DBInstanceIdentifier] != nil { + continue + } + if instance.AutomationMode == types.AutomationModeFull { + _, err = c.rds.ModifyDBInstance(ctx, &rds.ModifyDBInstanceInput{ + DBInstanceIdentifier: instance.DBInstanceIdentifier, + AutomationMode: types.AutomationModeAllPaused, + }) + if err != nil { + logger.Error(pkgerrors.Wrap(err, fmt.Sprintf("stop rds custom automation%s", *instance.DBInstanceIdentifier))) + break + } + + } + switch to { + case constants.OnStatus: + _, err = c.rds.StartDBInstance(ctx, &rds.StartDBInstanceInput{ + DBInstanceIdentifier: instance.DBInstanceIdentifier, + }) + if err != nil { + logger.Error(pkgerrors.Wrap(err, fmt.Sprintf("start db instance %s", *instance.DBInstanceIdentifier))) + } + case constants.OffStatus: + _, err = c.rds.StopDBInstance(ctx, &rds.StopDBInstanceInput{ + DBInstanceIdentifier: instance.DBInstanceIdentifier, + }) + if err != nil { + logger.Error(pkgerrors.Wrap(err, fmt.Sprintf("stop db instance %s", *instance.DBInstanceIdentifier))) + } + } + } + } + + return nil +} + +func (c *client) describeDBCluster(ctx context.Context) ([]types.DBCluster, error) { + dbClusters := []types.DBCluster{} + describeDBClusterOut, err := c.rds.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{}) + if err != nil { + return nil, err + } + dbClusters = append(dbClusters, describeDBClusterOut.DBClusters...) + for describeDBClusterOut.Marker != nil { + describeDBClusterOut, err := c.rds.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{Marker: describeDBClusterOut.Marker}) + if err != nil { + return nil, err + } + dbClusters = append(dbClusters, describeDBClusterOut.DBClusters...) + } + return dbClusters, nil +} + +func (c *client) describeDBInstance(ctx context.Context) ([]types.DBInstance, error) { + dbInstances := []types.DBInstance{} + describeDBInstanceOut, err := c.rds.DescribeDBInstances(ctx, &rds.DescribeDBInstancesInput{}) + if err != nil { + return nil, err + } + dbInstances = append(dbInstances, describeDBInstanceOut.DBInstances...) + for describeDBInstanceOut.Marker != nil { + describeDBInstanceOut, err := c.rds.DescribeDBInstances(ctx, &rds.DescribeDBInstancesInput{ + Marker: describeDBInstanceOut.Marker, + }) + if err != nil { + return nil, err + } + dbInstances = append(dbInstances, describeDBInstanceOut.DBInstances...) + } + return dbInstances, nil +} diff --git a/pkg/aws/rds/client_test.go b/pkg/aws/rds/client_test.go new file mode 100644 index 0000000..a987c52 --- /dev/null +++ b/pkg/aws/rds/client_test.go @@ -0,0 +1,22 @@ +package rds_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/nduyphuong/gorya/pkg/aws/rds" + "github.com/stretchr/testify/assert" +) + +func TestSmoke(t *testing.T) { + ctx := context.TODO() + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion("ap-southeast-1"), + ) + assert.NoError(t, err) + c, err := rds.NewFromConfig(cfg) + assert.NoError(t, err) + err = c.ChangeStatus(ctx, 1, "foo", "bar") + assert.NoError(t, err) +} diff --git a/pkg/gcp/client.go b/pkg/gcp/client.go index da5f15d..4dc5ecf 100644 --- a/pkg/gcp/client.go +++ b/pkg/gcp/client.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/nduyphuong/gorya/internal/constants" + "github.com/nduyphuong/gorya/pkg/gcp/cloudsql" "github.com/nduyphuong/gorya/pkg/gcp/gce" "github.com/nduyphuong/gorya/pkg/gcp/options" "google.golang.org/api/impersonate" @@ -14,11 +15,13 @@ import ( //go:generate mockery --name Interface type Interface interface { GCE() gce.Interface + CloudSQL() cloudsql.Interface } type client struct { - gce gce.Interface - opts options.Options + gce gce.Interface + cloudSql cloudsql.Interface + opts options.Options } type ClientPool struct { @@ -57,7 +60,7 @@ func (b *ClientPool) GetForCredential(name string) (Interface, bool) { if !ok { return nil, false } - fmt.Printf("got client from pool for %s", name) + fmt.Printf("got client from pool for %s\n", name) return i, true } @@ -81,9 +84,17 @@ func new(ctx context.Context, opts ...options.Option) (*client, error) { if err != nil { return nil, err } + c.cloudSql, err = cloudsql.NewService(ctx, &ts, options.WithProject(c.opts.Project)) + if err != nil { + return nil, err + } return &c, nil } func (c *client) GCE() gce.Interface { return c.gce } + +func (c *client) CloudSQL() cloudsql.Interface { + return c.cloudSql +} diff --git a/pkg/gcp/cloudsql/client.go b/pkg/gcp/cloudsql/client.go new file mode 100644 index 0000000..e2d23f8 --- /dev/null +++ b/pkg/gcp/cloudsql/client.go @@ -0,0 +1,88 @@ +package cloudsql + +import ( + "context" + "errors" + "sync" + + "github.com/nduyphuong/gorya/internal/constants" + "github.com/nduyphuong/gorya/internal/logging" + "github.com/nduyphuong/gorya/pkg/gcp/options" + "github.com/nduyphuong/gorya/pkg/gcp/utils" + pkgerrors "github.com/pkg/errors" + "golang.org/x/oauth2" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + sql "google.golang.org/api/sqladmin/v1beta4" +) + +var ErrInvalidResourceStatus = errors.New("invalid resource status") + +type Interface interface { + ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) +} + +type client struct { + sql *sql.Service + opts options.Options +} + +func NewService(ctx context.Context, ts *oauth2.TokenSource, opts ...options.Option) (*client, error) { + var err error + c := &client{} + for _, o := range opts { + o.Apply(&c.opts) + } + c.sql, err = sql.NewService(ctx, option.WithTokenSource(*ts)) + if err != nil { + return nil, err + } + return c, nil +} + +func (c *client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) error { + logger := logging.LoggerFromContext(ctx) + if to != constants.OffStatus && to != constants.OnStatus { + return ErrInvalidResourceStatus + } + var action string + if to == constants.OffStatus { + action = "NEVER" + } + if to == constants.OnStatus { + action = "ALWAYS" + } + tagFilter := utils.GetCloudSqlFilter(tagKey, tagValue) + instancesListResp, err := c.sql.Instances.List(c.opts.Project).Filter(tagFilter).Do() + if err != nil { + return pkgerrors.Wrap(err, "list instances") + } + replicasToInstance := map[string]string{} + var wg sync.WaitGroup + wg.Add(len(instancesListResp.Items)) + for _, instance := range instancesListResp.Items { + instance := instance + if len(instance.ReplicaNames) > 0 { + for _, replName := range instance.ReplicaNames { + replicasToInstance[replName] = instance.Name + } + } + if _, exist := replicasToInstance[instance.Name]; exist { + // instance is replica + continue + } + go func() { + defer wg.Done() + rb := &sql.DatabaseInstance{ + Settings: &sql.Settings{ + ActivationPolicy: action, + }, + } + _, err := c.sql.Instances.Patch(c.opts.Project, instance.Name, rb).Do() + if err != nil && !googleapi.IsNotModified(err) { + logger.Errorf("patch instance %s", instance.Name) + } + }() + } + return nil +} diff --git a/pkg/gcp/cloudsql/client_test.go b/pkg/gcp/cloudsql/client_test.go new file mode 100644 index 0000000..daee94c --- /dev/null +++ b/pkg/gcp/cloudsql/client_test.go @@ -0,0 +1,38 @@ +package cloudsql_test + +import ( + "context" + "testing" + + "github.com/nduyphuong/gorya/pkg/gcp/cloudsql" + "github.com/nduyphuong/gorya/pkg/gcp/options" + "github.com/stretchr/testify/assert" + "google.golang.org/api/impersonate" +) + +type TestData struct { + TargetPrincipal string + GCPProjectId string + Scopes []string +} + +func TestSmoke(t *testing.T) { + ctx := context.TODO() + d := TestData{ + TargetPrincipal: "priv-sa@target-project-397310.iam.gserviceaccount.com", + GCPProjectId: "target-project-397310", + Scopes: []string{ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + }, + } + ts, err := impersonate.CredentialsTokenSource(ctx, impersonate.CredentialsConfig{ + TargetPrincipal: d.TargetPrincipal, + Scopes: d.Scopes, + }) + assert.NoError(t, err) + sqlService, err := cloudsql.NewService(ctx, &ts, options.WithProject(d.GCPProjectId)) + assert.NoError(t, err) + err = sqlService.ChangeStatus(ctx, 0, "foo", "bar") + assert.NoError(t, err) +} diff --git a/pkg/gcp/gce/client.go b/pkg/gcp/gce/client.go index a8011af..06b53fb 100644 --- a/pkg/gcp/gce/client.go +++ b/pkg/gcp/gce/client.go @@ -3,9 +3,9 @@ package gce import ( "context" "errors" - "fmt" "github.com/nduyphuong/gorya/pkg/gcp/options" + "github.com/nduyphuong/gorya/pkg/gcp/utils" "golang.org/x/oauth2" compute "google.golang.org/api/compute/v1" "google.golang.org/api/option" @@ -16,14 +16,14 @@ type Interface interface { ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) (err error) } -type Client struct { +type client struct { gce *compute.Service opts options.Options } -func NewService(ctx context.Context, ts *oauth2.TokenSource, opts ...options.Option) (*Client, error) { +func NewService(ctx context.Context, ts *oauth2.TokenSource, opts ...options.Option) (*client, error) { var err error - c := &Client{} + c := &client{} for _, o := range opts { o.Apply(&c.opts) } @@ -34,7 +34,7 @@ func NewService(ctx context.Context, ts *oauth2.TokenSource, opts ...options.Opt return c, nil } -func (c *Client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) error { +func (c *client) ChangeStatus(ctx context.Context, to int, tagKey string, tagValue string) error { if to != 0 && to != 1 { return errors.New("to must have value of 0 or 1") } @@ -47,7 +47,7 @@ func (c *Client) ChangeStatus(ctx context.Context, to int, tagKey string, tagVal for _, zone := range zoneListResp.Items { zones = append(zones, zone.Description) } - tagFilter := fmt.Sprintf("labels.%s=%s", tagKey, tagValue) + tagFilter := utils.GetComputeFilter(tagKey, tagValue) for _, zone := range zones { instanceListResp, err := c.gce.Instances.List(c.opts.Project, zone).Context(ctx).Filter(tagFilter).Do() if err != nil { diff --git a/pkg/gcp/utils/filter.go b/pkg/gcp/utils/filter.go new file mode 100644 index 0000000..3f05476 --- /dev/null +++ b/pkg/gcp/utils/filter.go @@ -0,0 +1,11 @@ +package utils + +import "fmt" + +func GetComputeFilter(key, val string) string { + return fmt.Sprintf("labels.%s=%s", key, val) +} + +func GetCloudSqlFilter(key, val string) string { + return fmt.Sprintf("settings.userLabels.%s=%s", key, val) +}