From 35070f6b1fc635b1b81fb4488e5ba364c37c30c8 Mon Sep 17 00:00:00 2001 From: KH <96801545+kleineshertz@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:56:06 -0700 Subject: [PATCH] Move resources up to cld (#3) --- pkg/cld/cldaws/resources.go | 101 ++++++++++++------------------- pkg/cld/resource.go | 29 +++++++++ pkg/cmd/capideploy/capideploy.go | 54 ++++++++++++++--- pkg/provider/deploy_provider.go | 52 ++++++---------- 4 files changed, 129 insertions(+), 107 deletions(-) create mode 100644 pkg/cld/resource.go diff --git a/pkg/cld/cldaws/resources.go b/pkg/cld/cldaws/resources.go index a588dd6..1c71550 100644 --- a/pkg/cld/cldaws/resources.go +++ b/pkg/cld/cldaws/resources.go @@ -11,42 +11,17 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2/types" tagging "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" taggingTypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" + "github.com/capillariesio/capillaries-deploy/pkg/cld" "github.com/capillariesio/capillaries-deploy/pkg/l" ) -type ResourceBilledState string - -const ( - ResourceBilledStateUnknown ResourceBilledState = "unknown" - ResourceBilledStateActive ResourceBilledState = "active" - ResourceBilledStateTerminated ResourceBilledState = "terminated" -) - -const DeploymentNameTagName string = "DeploymentName" -const DeploymentOperatorTagName string = "DeploymentOperator" -const DeploymentOperatorTagValue string = "capideploy" - -type Resource struct { - DeploymentName string - Svc string - Type string - Id string - Name string - State string - BilledState ResourceBilledState -} - -func (r *Resource) String() string { - return fmt.Sprintf("%s, %s,%s,%s,%s,%s,%s", r.DeploymentName, r.Svc, r.Type, r.Name, r.Id, r.State, r.BilledState) -} - -func arnToResource(arn string) Resource { - r := Resource{ +func arnToResource(arn string) cld.Resource { + r := cld.Resource{ Svc: "unknown", Type: "unknown", Id: "unknown", State: "unknown", - BilledState: ResourceBilledStateUnknown, + BilledState: cld.ResourceBilledStateUnknown, } s := strings.Split(arn, "/") if len(s) >= 2 { @@ -62,50 +37,50 @@ func arnToResource(arn string) Resource { return r } -func getInstanceBilledState(state types.InstanceStateName) ResourceBilledState { +func getInstanceBilledState(state types.InstanceStateName) cld.ResourceBilledState { if state == types.InstanceStateNamePending || state == types.InstanceStateNameRunning { - return ResourceBilledStateActive + return cld.ResourceBilledStateActive } else { - return ResourceBilledStateTerminated + return cld.ResourceBilledStateTerminated } } -func getVolumeBilledState(state types.VolumeState) ResourceBilledState { +func getVolumeBilledState(state types.VolumeState) cld.ResourceBilledState { if state == types.VolumeStateAvailable || state == types.VolumeStateCreating || state == types.VolumeStateInUse { - return ResourceBilledStateActive + return cld.ResourceBilledStateActive } else { - return ResourceBilledStateTerminated + return cld.ResourceBilledStateTerminated } } -func getNatGatewayBilledState(state types.NatGatewayState) ResourceBilledState { +func getNatGatewayBilledState(state types.NatGatewayState) cld.ResourceBilledState { if state == types.NatGatewayStatePending || state == types.NatGatewayStateAvailable { - return ResourceBilledStateActive + return cld.ResourceBilledStateActive } else { - return ResourceBilledStateTerminated + return cld.ResourceBilledStateTerminated } } -func getVpcBilledState(state types.VpcState) ResourceBilledState { +func getVpcBilledState(state types.VpcState) cld.ResourceBilledState { if state == types.VpcStatePending || state == types.VpcStateAvailable { - return ResourceBilledStateActive + return cld.ResourceBilledStateActive } else { - return ResourceBilledStateTerminated + return cld.ResourceBilledStateTerminated } } -func getImageBilledState(state types.ImageState) ResourceBilledState { +func getImageBilledState(state types.ImageState) cld.ResourceBilledState { if state == types.ImageStateAvailable || state == types.ImageStateDisabled || state == types.ImageStateError || state == types.ImageStatePending || state == types.ImageStateTransient { - return ResourceBilledStateActive + return cld.ResourceBilledStateActive } else { - return ResourceBilledStateTerminated + return cld.ResourceBilledStateTerminated } } -func getSnapshotBilledState(_ types.SnapshotState) ResourceBilledState { - return ResourceBilledStateActive +func getSnapshotBilledState(_ types.SnapshotState) cld.ResourceBilledState { + return cld.ResourceBilledStateActive } -func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) (string, ResourceBilledState, error) { +func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *cld.Resource) (string, cld.ResourceBilledState, error) { switch r.Svc { case "ec2": switch r.Type { @@ -114,7 +89,7 @@ func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) if err != nil { return "", "", err } - return *out.Addresses[0].PublicIp, ResourceBilledStateActive, nil + return *out.Addresses[0].PublicIp, cld.ResourceBilledStateActive, nil case "vpc": out, err := ec2Client.DescribeVpcs(goCtx, &ec2.DescribeVpcsInput{VpcIds: []string{r.Id}}) if err != nil { @@ -126,36 +101,36 @@ func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) if err != nil { return "", "", err } - return string(out.Subnets[0].State), ResourceBilledStateActive, nil + return string(out.Subnets[0].State), cld.ResourceBilledStateActive, nil case "security-group": _, err := ec2Client.DescribeSecurityGroups(goCtx, &ec2.DescribeSecurityGroupsInput{GroupIds: []string{r.Id}}) if err != nil { return "", "", err } - return "present", ResourceBilledStateActive, nil + return "present", cld.ResourceBilledStateActive, nil case "route-table": out, err := ec2Client.DescribeRouteTables(goCtx, &ec2.DescribeRouteTablesInput{RouteTableIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "does not exist") { - return "doesnotexist", ResourceBilledStateTerminated, nil + return "doesnotexist", cld.ResourceBilledStateTerminated, nil } return "", "", err } - return fmt.Sprintf("%droutes", len(out.RouteTables[0].Routes)), ResourceBilledStateActive, nil + return fmt.Sprintf("%droutes", len(out.RouteTables[0].Routes)), cld.ResourceBilledStateActive, nil case "instance": out, err := ec2Client.DescribeInstances(goCtx, &ec2.DescribeInstancesInput{InstanceIds: []string{r.Id}}) if err != nil { return "", "", err } if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 { - return "notfound", ResourceBilledStateTerminated, nil + return "notfound", cld.ResourceBilledStateTerminated, nil } return string(out.Reservations[0].Instances[0].State.Name), getInstanceBilledState(out.Reservations[0].Instances[0].State.Name), nil case "volume": out, err := ec2Client.DescribeVolumes(goCtx, &ec2.DescribeVolumesInput{VolumeIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "does not exist") { - return "doesnotexist", ResourceBilledStateTerminated, nil + return "doesnotexist", cld.ResourceBilledStateTerminated, nil } return "", "", err } @@ -164,7 +139,7 @@ func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) out, err := ec2Client.DescribeNatGateways(goCtx, &ec2.DescribeNatGatewaysInput{NatGatewayIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "was not found") { - return "notfound", ResourceBilledStateTerminated, nil + return "notfound", cld.ResourceBilledStateTerminated, nil } return "", "", err } @@ -173,16 +148,16 @@ func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) out, err := ec2Client.DescribeInternetGateways(goCtx, &ec2.DescribeInternetGatewaysInput{InternetGatewayIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "does not exist") { - return "doesnotexist", ResourceBilledStateTerminated, nil + return "doesnotexist", cld.ResourceBilledStateTerminated, nil } return "", "", err } - return fmt.Sprintf("%dattachments", len(out.InternetGateways[0].Attachments)), ResourceBilledStateActive, nil + return fmt.Sprintf("%dattachments", len(out.InternetGateways[0].Attachments)), cld.ResourceBilledStateActive, nil case "image": out, err := ec2Client.DescribeImages(goCtx, &ec2.DescribeImagesInput{ImageIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "does not exist") { - return "doesnotexist", ResourceBilledStateTerminated, nil + return "doesnotexist", cld.ResourceBilledStateTerminated, nil } return "", "", err } @@ -192,7 +167,7 @@ func getResourceState(ec2Client *ec2.Client, goCtx context.Context, r *Resource) out, err := ec2Client.DescribeSnapshots(goCtx, &ec2.DescribeSnapshotsInput{SnapshotIds: []string{r.Id}}) if err != nil { if strings.Contains(err.Error(), "does not exist") { - return "doesnotexist", ResourceBilledStateTerminated, nil + return "doesnotexist", cld.ResourceBilledStateTerminated, nil } return "", "", err } @@ -216,15 +191,15 @@ func getResourceDeploymentNameAndNameTags(ec2Client *ec2.Client, goCtx context.C for _, tagDesc := range out.Tags { if *tagDesc.Key == "Name" { resourceNameTagValue = *tagDesc.Value - } else if *tagDesc.Key == DeploymentNameTagName { + } else if *tagDesc.Key == cld.DeploymentNameTagName { deploymentNameTagValue = *tagDesc.Value } } return deploymentNameTagValue, resourceNameTagValue, nil } -func GetResourcesByTag(tClient *tagging.Client, ec2Client *ec2.Client, goCtx context.Context, lb *l.LogBuilder, region string, tagFilters []taggingTypes.TagFilter, readState bool) ([]*Resource, error) { - resources := make([]*Resource, 0) +func GetResourcesByTag(tClient *tagging.Client, ec2Client *ec2.Client, goCtx context.Context, lb *l.LogBuilder, region string, tagFilters []taggingTypes.TagFilter, readState bool) ([]*cld.Resource, error) { + resources := make([]*cld.Resource, 0) paginationToken := "" for { out, err := tClient.GetResources(goCtx, &tagging.GetResourcesInput{ @@ -232,7 +207,7 @@ func GetResourcesByTag(tClient *tagging.Client, ec2Client *ec2.Client, goCtx con PaginationToken: &paginationToken, TagFilters: tagFilters}) if err != nil { - return []*Resource{}, err + return []*cld.Resource{}, err } for _, rtMapping := range out.ResourceTagMappingList { diff --git a/pkg/cld/resource.go b/pkg/cld/resource.go new file mode 100644 index 0000000..760947e --- /dev/null +++ b/pkg/cld/resource.go @@ -0,0 +1,29 @@ +package cld + +import "fmt" + +type ResourceBilledState string + +const ( + ResourceBilledStateUnknown ResourceBilledState = "unknown" + ResourceBilledStateActive ResourceBilledState = "active" + ResourceBilledStateTerminated ResourceBilledState = "terminated" +) + +const DeploymentNameTagName string = "DeploymentName" +const DeploymentOperatorTagName string = "DeploymentOperator" +const DeploymentOperatorTagValue string = "capideploy" + +type Resource struct { + DeploymentName string + Svc string + Type string + Id string + Name string + State string + BilledState ResourceBilledState +} + +func (r *Resource) String() string { + return fmt.Sprintf("%s, %s,%s,%s,%s,%s,%s", r.DeploymentName, r.Svc, r.Type, r.Name, r.Id, r.State, r.BilledState) +} diff --git a/pkg/cmd/capideploy/capideploy.go b/pkg/cmd/capideploy/capideploy.go index 1fe221f..398128c 100644 --- a/pkg/cmd/capideploy/capideploy.go +++ b/pkg/cmd/capideploy/capideploy.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/capillariesio/capillaries-deploy/pkg/cld" "github.com/capillariesio/capillaries-deploy/pkg/l" "github.com/capillariesio/capillaries-deploy/pkg/prj" "github.com/capillariesio/capillaries-deploy/pkg/provider" @@ -259,14 +260,12 @@ func main() { var prjErr error singleThreadCommands := map[string]SingleThreadCmdHandler{ - CmdListDeployments: nil, - CmdListDeploymentResources: nil, - CmdCreateFloatingIps: nil, - CmdDeleteFloatingIps: nil, - CmdCreateSecurityGroups: nil, - CmdDeleteSecurityGroups: nil, - CmdCreateNetworking: nil, - CmdDeleteNetworking: nil, + CmdCreateFloatingIps: nil, + CmdDeleteFloatingIps: nil, + CmdCreateSecurityGroups: nil, + CmdDeleteSecurityGroups: nil, + CmdCreateNetworking: nil, + CmdDeleteNetworking: nil, } if _, ok := singleThreadCommands[os.Args[1]]; ok { @@ -287,8 +286,43 @@ func main() { if deployProviderErr != nil { log.Fatalf(deployProviderErr.Error()) } - singleThreadCommands[CmdListDeployments] = deployProvider.ListDeployments - singleThreadCommands[CmdListDeploymentResources] = deployProvider.ListDeploymentResources + + if os.Args[1] == CmdListDeployments { + mapResourceCount, logMsg, err := deployProvider.ListDeployments() + fmt.Println(logMsg) + if err != nil { + log.Fatalf(err.Error()) + } + + deploymentStrings := make([]string, len(mapResourceCount)) + deploymentIdx := 0 + totalResourceCount := 0 + for deploymentName, deploymentResCount := range mapResourceCount { + deploymentStrings[deploymentIdx] = fmt.Sprintf("%s,%d", deploymentName, deploymentResCount) + deploymentIdx++ + totalResourceCount += deploymentResCount + } + fmt.Printf("%s\n", strings.Join(deploymentStrings, "\n")) + fmt.Printf("Deployments: %d, resources: %d\n", len(mapResourceCount), totalResourceCount) + os.Exit(0) + } else if os.Args[1] == CmdListDeploymentResources { + resources, logMsg, err := deployProvider.ListDeploymentResources() + fmt.Println(logMsg) + if err != nil { + log.Fatalf(err.Error()) + } + resourceStrings := make([]string, len(resources)) + activeCount := 0 + for resIdx, res := range resources { + resourceStrings[resIdx] = res.String() + if res.BilledState != cld.ResourceBilledStateTerminated { + activeCount++ + } + } + fmt.Printf("%s\n", strings.Join(resourceStrings, "\n")) + fmt.Printf("Total: %d, potentially billed: %d\n", len(resources), activeCount) + } + singleThreadCommands[CmdCreateFloatingIps] = deployProvider.CreateFloatingIps singleThreadCommands[CmdDeleteFloatingIps] = deployProvider.DeleteFloatingIps singleThreadCommands[CmdCreateSecurityGroups] = deployProvider.CreateSecurityGroups diff --git a/pkg/provider/deploy_provider.go b/pkg/provider/deploy_provider.go index 5d56994..978c4cd 100644 --- a/pkg/provider/deploy_provider.go +++ b/pkg/provider/deploy_provider.go @@ -3,7 +3,6 @@ package provider import ( "context" "fmt" - "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -12,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" taggingTypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/capillariesio/capillaries-deploy/pkg/cld" "github.com/capillariesio/capillaries-deploy/pkg/cld/cldaws" "github.com/capillariesio/capillaries-deploy/pkg/l" "github.com/capillariesio/capillaries-deploy/pkg/prj" @@ -33,8 +33,8 @@ type DeployCtx struct { } type DeployProvider interface { GetCtx() *DeployCtx - ListDeployments() (l.LogMsg, error) - ListDeploymentResources() (l.LogMsg, error) + ListDeployments() (map[string]int, l.LogMsg, error) + ListDeploymentResources() ([]*cld.Resource, l.LogMsg, error) CreateFloatingIps() (l.LogMsg, error) DeleteFloatingIps() (l.LogMsg, error) CreateSecurityGroups() (l.LogMsg, error) @@ -91,8 +91,8 @@ func DeployProviderFactory(project *prj.Project, goCtx context.Context, assumeRo GoCtx: goCtx, IsVerbose: isVerbose, Tags: map[string]string{ - cldaws.DeploymentNameTagName: project.DeploymentName, - cldaws.DeploymentOperatorTagName: cldaws.DeploymentOperatorTagValue}, + cld.DeploymentNameTagName: project.DeploymentName, + cld.DeploymentOperatorTagName: cld.DeploymentOperatorTagValue}, Aws: &AwsCtx{ Ec2Client: ec2.NewFromConfig(cfg), TaggingClient: resourcegroupstaggingapi.NewFromConfig(cfg), @@ -123,12 +123,13 @@ export BASTION_IP=%s prj.SshConfig.PrivateKeyPath, prj.SshConfig.BastionExternalIp) } -func (p *AwsDeployProvider) ListDeployments() (l.LogMsg, error) { +func (p *AwsDeployProvider) ListDeployments() (map[string]int, l.LogMsg, error) { lb := l.NewLogBuilder(l.CurFuncName(), p.GetCtx().IsVerbose) resources, err := cldaws.GetResourcesByTag(p.GetCtx().Aws.TaggingClient, p.GetCtx().Aws.Ec2Client, p.GetCtx().GoCtx, lb, p.GetCtx().Aws.Config.Region, - []taggingTypes.TagFilter{{Key: aws.String(cldaws.DeploymentOperatorTagName), Values: []string{cldaws.DeploymentOperatorTagValue}}}, false) + []taggingTypes.TagFilter{{Key: aws.String(cld.DeploymentOperatorTagName), Values: []string{cld.DeploymentOperatorTagValue}}}, false) if err != nil { - return lb.Complete(err) + logMsg, err := lb.Complete(err) + return nil, logMsg, err } deploymentResCount := map[string]int{} for _, res := range resources { @@ -138,37 +139,20 @@ func (p *AwsDeployProvider) ListDeployments() (l.LogMsg, error) { deploymentResCount[res.DeploymentName] = 1 } } - deploymentStrings := make([]string, len(deploymentResCount)) - deploymentIdx := 0 - totalResourceCount := 0 - for deploymentName, deploymentResCount := range deploymentResCount { - deploymentStrings[deploymentIdx] = fmt.Sprintf("%s,%d", deploymentName, deploymentResCount) - deploymentIdx++ - totalResourceCount += deploymentResCount - } - fmt.Printf("%s\n", strings.Join(deploymentStrings, "\n")) - fmt.Printf("Deployments: %d, resources: %d\n", len(deploymentResCount), totalResourceCount) - return lb.Complete(nil) + logMsg, _ := lb.Complete(nil) + return deploymentResCount, logMsg, nil } -func (p *AwsDeployProvider) ListDeploymentResources() (l.LogMsg, error) { +func (p *AwsDeployProvider) ListDeploymentResources() ([]*cld.Resource, l.LogMsg, error) { lb := l.NewLogBuilder(l.CurFuncName(), p.GetCtx().IsVerbose) resources, err := cldaws.GetResourcesByTag(p.GetCtx().Aws.TaggingClient, p.GetCtx().Aws.Ec2Client, p.GetCtx().GoCtx, lb, p.GetCtx().Aws.Config.Region, []taggingTypes.TagFilter{ - {Key: aws.String(cldaws.DeploymentOperatorTagName), Values: []string{cldaws.DeploymentOperatorTagValue}}, - {Key: aws.String(cldaws.DeploymentNameTagName), Values: []string{p.Ctx.Project.DeploymentName}}}, true) + {Key: aws.String(cld.DeploymentOperatorTagName), Values: []string{cld.DeploymentOperatorTagValue}}, + {Key: aws.String(cld.DeploymentNameTagName), Values: []string{p.Ctx.Project.DeploymentName}}}, true) if err != nil { - return lb.Complete(err) - } - resourceStrings := make([]string, len(resources)) - activeCount := 0 - for resIdx, res := range resources { - resourceStrings[resIdx] = res.String() - if res.BilledState != cldaws.ResourceBilledStateTerminated { - activeCount++ - } + logMsg, err := lb.Complete(err) + return nil, logMsg, err } - fmt.Printf("%s\n", strings.Join(resourceStrings, "\n")) - fmt.Printf("Total: %d, potentially billed: %d\n", len(resources), activeCount) - return lb.Complete(nil) + logMsg, _ := lb.Complete(nil) + return resources, logMsg, nil }