From 8daf59aeb0e077c17dfa3bb64972ee025d880d3b Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 28 May 2021 14:10:01 +0200 Subject: [PATCH 01/24] #SOS-15 /collector/structs.go + AccountSpecifiedFiels struct to save AccountId and AccountName /collector/aws/common/structs.go + add Method getAccountName() /collector/aws/detector.go + save in DetectorManager the accountName from config file /collector/resources/* + save AccountSpecifiedFields with values in detectedStructs --- collector/aws/common/structs.go | 1 + collector/aws/detector.go | 17 ++++++++++++++++- collector/aws/resources/apigateway.go | 5 +++++ collector/aws/resources/docdb.go | 5 +++++ collector/aws/resources/dynamodb.go | 5 +++++ collector/aws/resources/ec2.go | 5 +++++ collector/aws/resources/ec2volumes.go | 5 +++++ collector/aws/resources/elasticache.go | 5 +++++ collector/aws/resources/elasticips.go | 5 +++++ collector/aws/resources/elasticsearch.go | 5 +++++ collector/aws/resources/elb.go | 5 +++++ collector/aws/resources/elbv2.go | 5 +++++ collector/aws/resources/iam.go | 5 +++++ collector/aws/resources/kinesis.go | 5 +++++ collector/aws/resources/lambda.go | 5 +++++ collector/aws/resources/natgateway.go | 5 +++++ collector/aws/resources/neptune.go | 5 +++++ collector/aws/resources/rds.go | 5 +++++ collector/aws/resources/redshift.go | 5 +++++ collector/structs.go | 6 ++++++ 20 files changed, 108 insertions(+), 1 deletion(-) diff --git a/collector/aws/common/structs.go b/collector/aws/common/structs.go index ebd7a22b..cb3241c5 100644 --- a/collector/aws/common/structs.go +++ b/collector/aws/common/structs.go @@ -26,6 +26,7 @@ type AWSManager interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *aws.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput SetGlobal(resourceName collector.ResourceIdentifier) diff --git a/collector/aws/detector.go b/collector/aws/detector.go index 0450dfa9..857ae7f6 100644 --- a/collector/aws/detector.go +++ b/collector/aws/detector.go @@ -21,6 +21,7 @@ type DetectorDescriptor interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *awsClient.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput } @@ -38,12 +39,20 @@ type DetectorManager struct { session *session.Session awsConfig *awsClient.Config accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } // NewDetectorManager create new instance of detector manager -func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDescriber, account config.AWSAccount, stsManager *STSManager, global map[string]struct{}, region string) *DetectorManager { +func NewDetectorManager( + awsAuth AuthDescriptor, + collector collector.CollectorDescriber, + account config.AWSAccount, + stsManager *STSManager, + global map[string]struct{}, + region string, +) *DetectorManager { priceSession, _ := awsAuth.Login(defaultRegionPrice) pricingManager := pricing.NewPricingManager(awsPricing.New(priceSession), defaultRegionPrice) @@ -60,6 +69,7 @@ func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDes session: regionSession, awsConfig: regionConfig, accountIdentity: callerIdentityOutput, + accountName: account.Name, global: global, } } @@ -89,6 +99,11 @@ func (dm *DetectorManager) GetRegion() string { return dm.region } +// GetAccountName returns the account name +func (dm *DetectorManager) GetAccountName() string { + return dm.accountName +} + // GetSession return the aws session func (dm *DetectorManager) GetSession() (*session.Session, *awsClient.Config) { return dm.session, dm.awsConfig diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index 0157e07d..e4fc5af2 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -36,6 +36,7 @@ type DetectedAPIGateway struct { Name string LaunchTime time.Time Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -147,6 +148,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, Name: *api.Name, LaunchTime: *api.CreatedDate, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }, } ag.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index 5753c94f..f6caabec 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -39,6 +39,7 @@ type DetectedDocumentDB struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -162,6 +163,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/dynamodb.go b/collector/aws/resources/dynamodb.go index 6c109980..8b08b706 100644 --- a/collector/aws/resources/dynamodb.go +++ b/collector/aws/resources/dynamodb.go @@ -42,6 +42,7 @@ type DetectedAWSDynamoDB struct { Metric string Name string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -199,6 +200,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e PricePerMonth: pricePerMonth, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 1bba1697..46663bbc 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -38,6 +38,7 @@ type DetectedEC2 struct { Name string InstanceType string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -164,6 +165,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index ac5e0fa3..a6db7710 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -36,6 +36,7 @@ type DetectedAWSEC2Volume struct { Size int64 PricePerMonth float64 Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -124,6 +125,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, Size: volumeSize, PricePerMonth: ev.getCalculatedPrice(vol, price), Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }, } ev.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index 473432ab..f0afc540 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -39,6 +39,7 @@ type DetectedElasticache struct { CacheNodeType string CacheNodes int collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -163,6 +164,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index b22b94fd..ba43edef 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -35,6 +35,7 @@ type DetectedElasticIP struct { PricePerHour float64 PricePerMonth float64 Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -115,6 +116,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }, } ei.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticsearch.go b/collector/aws/resources/elasticsearch.go index 03e4e797..433beeb1 100644 --- a/collector/aws/resources/elasticsearch.go +++ b/collector/aws/resources/elasticsearch.go @@ -46,6 +46,7 @@ type DetectedElasticSearch struct { InstanceType string InstanceCount int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } // elasticSearchVolumeType will hold the available volume types for ESCluster EBS @@ -220,6 +221,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac PricePerMonth: hourlyClusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }, } esm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 4ef6cfec..11a9594e 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -37,6 +37,7 @@ type DetectedELB struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -176,6 +177,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index c74ed42e..906a272c 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -38,6 +38,7 @@ type DetectedELBV2 struct { Region string Type string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // loadBalancerConfig defines loadbalancer's configuration of metrics and pricing @@ -219,6 +220,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index 4aa8ca20..e2845024 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -34,6 +34,7 @@ type DetectedAWSLastActivity struct { AccessKey string LastUsedDate time.Time LastActivity string + collector.AccountSpecifiedFields } func init() { @@ -133,6 +134,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) AccessKey: *accessKeyData.AccessKeyId, LastUsedDate: lastUsedDate, LastActivity: lastActivity, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }, } im.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index f1cd3d1c..6f30c255 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -38,6 +38,7 @@ type DetectedKinesis struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -193,6 +194,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er PricePerMonth: totalShardsPerHourPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }, } km.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/lambda.go b/collector/aws/resources/lambda.go index e6e2569f..e22f96e5 100644 --- a/collector/aws/resources/lambda.go +++ b/collector/aws/resources/lambda.go @@ -36,6 +36,7 @@ type DetectedAWSLambda struct { ResourceID string Name string Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -149,6 +150,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err ResourceID: *fun.FunctionArn, Name: *fun.FunctionName, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }, } lm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index a4f788e5..c15d1fc8 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -39,6 +39,7 @@ type DetectedNATGateway struct { SubnetID string VPCID string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -178,6 +179,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }, } ngw.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/neptune.go b/collector/aws/resources/neptune.go index 7a22ad3f..7092eee0 100644 --- a/collector/aws/resources/neptune.go +++ b/collector/aws/resources/neptune.go @@ -39,6 +39,7 @@ type DetectedAWSNeptune struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -163,6 +164,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }, } np.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index ba2d5d67..6c9ee1a4 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -43,6 +43,7 @@ type DetectedAWSRDS struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // RDSVolumeType will hold the available volume types for RDS types @@ -206,6 +207,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: totalHourlyPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }, } r.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index 7f660d6e..dc709e52 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -38,6 +38,7 @@ type DetectedRedShift struct { NodeType string NumberOfNodes int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -159,6 +160,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerMonth: clusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }, } rdm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/structs.go b/collector/structs.go index 6ecd26f4..58119e3b 100644 --- a/collector/structs.go +++ b/collector/structs.go @@ -36,6 +36,12 @@ type PriceDetectedFields struct { Tag map[string]string } +// AccountSpecifiedFields describe account data of an resource +type AccountSpecifiedFields struct { + AccountID string + AccountName string +} + // EventCollector collector event data structure type EventCollector struct { EventType string From 35a3f71bac52f904b61e69e29a706a6d55e6c485 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Sat, 29 May 2021 15:53:57 +0200 Subject: [PATCH 02/24] #SOS-22 * elasticsearch.go getDynamicMatchQuery add: if filterName is Data.AccountID then build boolQuery for given AccountIDs --- api/storage/elasticsearch/elasticsearch.go | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index afda70a2..7fee0744 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -10,6 +10,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "time" elastic "github.com/olivere/elastic/v7" @@ -110,14 +111,24 @@ func (sm *StorageManager) getDynamicMatchQuery(filters map[string]string, operat dynamicMatchQuery := []elastic.Query{} var mq *elastic.MatchQuery for name, value := range filters { - mq = elastic.NewMatchQuery(name, value) - // Minimum number of clauses that must match for a document to be returned - mq.MinimumShouldMatch("100%") - if operator == "and" { - mq = mq.Operator("and") + if name == "Data.AccountID" { + var accountIds = strings.Split(value, ",") + var accountBoolQuery = elastic.NewBoolQuery() + for _, accountId := range accountIds { + accountBoolQuery.Should(elastic.NewMatchQuery(name, accountId)) + } + accountBoolQuery.MinimumShouldMatch("1") + dynamicMatchQuery = append(dynamicMatchQuery, accountBoolQuery) + } else { + mq = elastic.NewMatchQuery(name, value) + // Minimum number of clauses that must match for a document to be returned + mq.MinimumShouldMatch("100%") + if operator == "and" { + mq = mq.Operator("and") + } + log.Info("Query ", mq) + dynamicMatchQuery = append(dynamicMatchQuery, mq) } - - dynamicMatchQuery = append(dynamicMatchQuery, mq) } return dynamicMatchQuery } From 5866b6e615d9f3a20d7cabf8a6175e39182870ed Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 31 May 2021 15:05:44 +0200 Subject: [PATCH 03/24] #SOS-15 fit MockAWSManager to the modified AWSManager --- collector/aws/testutils/detector.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/collector/aws/testutils/detector.go b/collector/aws/testutils/detector.go index 4bab9eca..473e9385 100644 --- a/collector/aws/testutils/detector.go +++ b/collector/aws/testutils/detector.go @@ -17,11 +17,17 @@ type MockAWSManager struct { pricing *pricing.PricingManager session *session.Session accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } -func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudwatch.CloudwatchManager, priceClient *pricing.PricingManager, region string) *MockAWSManager { +func AWSManager( + collector collector.CollectorDescriber, + cloudWatchClient *cloudwatch.CloudwatchManager, + priceClient *pricing.PricingManager, + region string, +) *MockAWSManager { accountID := "1234" accountIdentity := &sts.GetCallerIdentityOutput{ @@ -34,6 +40,7 @@ func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudw pricing: priceClient, accountIdentity: accountIdentity, region: region, + accountName: "test", global: make(map[string]struct{}), } } @@ -58,6 +65,10 @@ func (dm *MockAWSManager) GetRegion() string { return dm.region } +func (dm *MockAWSManager) GetAccountName() string { + return dm.accountName +} + func (dm *MockAWSManager) GetSession() (*session.Session, *aws.Config) { return dm.session, &aws.Config{} } From 839ae2febccf74e4f9a601bacd8e377dc74e9178 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Fri, 4 Jun 2021 20:22:31 +0200 Subject: [PATCH 04/24] #SOS-23 Add getAccounts + save account information in event_status --- api/route.go | 11 ++++++++ api/server.go | 1 + api/storage/elasticsearch/elasticsearch.go | 32 ++++++++++++++++++++++ api/storage/structs.go | 6 ++++ collector/aws/resources/apigateway.go | 10 +++++-- collector/aws/resources/docdb.go | 10 +++++-- collector/aws/resources/dynamodb.go | 10 +++++-- collector/aws/resources/ec2.go | 10 +++++-- collector/aws/resources/ec2volumes.go | 10 +++++-- collector/aws/resources/elasticache.go | 10 +++++-- collector/aws/resources/elasticips.go | 10 +++++-- collector/aws/resources/elasticsearch.go | 10 +++++-- collector/aws/resources/elb.go | 10 +++++-- collector/aws/resources/elbv2.go | 10 +++++-- collector/aws/resources/iam.go | 10 +++++-- collector/aws/resources/kinesis.go | 10 +++++-- collector/aws/resources/lambda.go | 10 +++++-- collector/aws/resources/natgateway.go | 10 +++++-- collector/aws/resources/neptune.go | 10 +++++-- collector/aws/resources/rds.go | 10 +++++-- collector/aws/resources/redshift.go | 10 +++++-- collector/collector.go | 14 ++++++---- collector/structs.go | 5 ++-- interpolation/interpolation.go | 9 ++++++ 24 files changed, 206 insertions(+), 42 deletions(-) diff --git a/api/route.go b/api/route.go index 2085741a..73357bc1 100644 --- a/api/route.go +++ b/api/route.go @@ -55,6 +55,17 @@ func (server *Server) GetExecutions(resp http.ResponseWriter, req *http.Request) server.JSONWrite(resp, http.StatusOK, results) } +func (server *Server) GetAccounts(resp http.ResponseWriter, req *http.Request) { + queryLimit, _ := strconv.Atoi(httpparameters.QueryParamWithDefault(req, "querylimit", storage.GetExecutionsQueryLimit)) + accounts, err := server.storage.GetAccounts(queryLimit) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + + } + server.JSONWrite(resp, http.StatusOK, accounts) +} + // GetResourceData return resuts details by resource type func (server *Server) GetResourceData(resp http.ResponseWriter, req *http.Request) { queryParams := req.URL.Query() diff --git a/api/server.go b/api/server.go index b82d45b1..10ad2498 100644 --- a/api/server.go +++ b/api/server.go @@ -81,6 +81,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") + server.router.HandleFunc("/api/v1/accounts", server.GetAccounts).Methods(("GET")) server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 7fee0744..9dcee359 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -309,6 +309,38 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e return executions, nil } +func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { + accounts := []storage.Accounts{} + + searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation().Field("Data.AccountInformation.keyword")). + Do(context.Background()) + + if err != nil { + log.WithError(err).Error("error when trying to get AccountIDs") + return accounts, ErrInvalidQuery + } + + resp, ok := searchResult.Aggregations.Terms("Accounts") + if !ok { + log.Error("accounts field term does not exist") + return accounts, ErrAggregationTermNotFound + } + + for _, accountsBucket := range resp.Buckets { + account := accountsBucket.Key.(string) + name, id, err := interpolation.ExtractAccountInformation(account) + if err != nil { + log.WithError(err).WithField("account", account).Error("could not extract account information") + continue + } + accounts = append(accounts, storage.Accounts{ + ID: id, + Name: name, + }) + } + return accounts, nil +} + // GetResources return resource data func (sm *StorageManager) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { diff --git a/api/storage/structs.go b/api/storage/structs.go index 467b8168..4d211301 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -13,6 +13,7 @@ type StorageDescriber interface { Save(data string) bool GetSummary(executionID string, filters map[string]string) (map[string]CollectorsSummary, error) GetExecutions(querylimit int) ([]Executions, error) + GetAccounts(querylimit int) ([]Accounts, error) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) GetResourceTrends(resourceType string, filters map[string]string, limit int) ([]ExecutionCost, error) GetExecutionTags(executionID string) (map[string][]string, error) @@ -31,6 +32,11 @@ type ExecutionCost struct { CostSum float64 } +type Accounts struct { + ID string + Name string +} + // CollectorsSummary defines unused resource summary type CollectorsSummary struct { ResourceName string `json:"ResourceName"` diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index e4fc5af2..118e5c65 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -72,7 +72,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "apigateway", }).Info("starting to analyze resource") - ag.awsManager.GetCollector().CollectStart(ag.Name) + ag.awsManager.GetCollector().CollectStart(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) detectAPIGateway := []DetectedAPIGateway{} apigateways, err := ag.getRestApis(nil, nil) @@ -165,7 +168,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ag.awsManager.GetCollector().CollectFinish(ag.Name) + ag.awsManager.GetCollector().CollectFinish(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) return detectAPIGateway, nil } diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index f6caabec..1929dde3 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -75,7 +75,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "documentDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedDocDB := []DetectedDocumentDB{} instances, err := dd.describeInstances(nil, nil) @@ -181,7 +184,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedDocDB, nil diff --git a/collector/aws/resources/dynamodb.go b/collector/aws/resources/dynamodb.go index 8b08b706..084b1260 100644 --- a/collector/aws/resources/dynamodb.go +++ b/collector/aws/resources/dynamodb.go @@ -79,7 +79,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e "resource": "dynamoDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedTables := []DetectedAWSDynamoDB{} tables, err := dd.describeTables(nil, nil) @@ -217,7 +220,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e } } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedTables, nil diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 46663bbc..52fe9f70 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -74,7 +74,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "ec2_instances", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedEC2 := []DetectedEC2{} @@ -183,7 +186,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedEC2, nil diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index a6db7710..b4a5ccc0 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -76,7 +76,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "ec2_volume", }).Info("starting to analyze resource") - ev.awsManager.GetCollector().CollectStart(ev.Name) + ev.awsManager.GetCollector().CollectStart(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) detected := []DetectedAWSEC2Volume{} volumes, err := ev.describe(nil, nil) @@ -140,7 +143,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } - ev.awsManager.GetCollector().CollectFinish(ev.Name) + ev.awsManager.GetCollector().CollectFinish(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index f0afc540..96c9c4c7 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -75,7 +75,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "elasticache", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedelasticache := []DetectedElasticache{} @@ -180,7 +183,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedelasticache, nil } diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index ba43edef..27336e3c 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -73,7 +73,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "elastic ips", }).Info("starting to analyze resource") - ei.awsManager.GetCollector().CollectStart(ei.Name) + ei.awsManager.GetCollector().CollectStart(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) elasticIPs := []DetectedElasticIP{} @@ -132,7 +135,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ei.awsManager.GetCollector().CollectFinish(ei.Name) + ei.awsManager.GetCollector().CollectFinish(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) return elasticIPs, nil diff --git a/collector/aws/resources/elasticsearch.go b/collector/aws/resources/elasticsearch.go index 433beeb1..13a7fb38 100644 --- a/collector/aws/resources/elasticsearch.go +++ b/collector/aws/resources/elasticsearch.go @@ -89,7 +89,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac "resource": "elasticsearch", }).Info("analyzing resource") - esm.awsManager.GetCollector().CollectStart(esm.Name) + esm.awsManager.GetCollector().CollectStart(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) detectedElasticSearchClusters := []DetectedElasticSearch{} @@ -237,7 +240,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac } } - esm.awsManager.GetCollector().CollectFinish(esm.Name) + esm.awsManager.GetCollector().CollectFinish(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) return detectedElasticSearchClusters, nil } diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 11a9594e..14b77d1c 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -73,7 +73,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "elb", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELB := []DetectedELB{} @@ -195,7 +198,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELB, nil diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index 906a272c..4bc5e2b4 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -104,7 +104,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro "resource": "elb_v2", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELBV2 := []DetectedELBV2{} @@ -236,7 +239,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELBV2, nil diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index e2845024..dcd12e66 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -76,7 +76,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "iam", }).Info("starting to analyze resource") - im.awsManager.GetCollector().CollectStart(im.Name) + im.awsManager.GetCollector().CollectStart(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) detected := []DetectedAWSLastActivity{} @@ -151,7 +154,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - im.awsManager.GetCollector().CollectFinish(im.Name) + im.awsManager.GetCollector().CollectFinish(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 6f30c255..b5471346 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -75,7 +75,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "kinesis", }).Info("analyzing resource") - km.awsManager.GetCollector().CollectStart(km.Name) + km.awsManager.GetCollector().CollectStart(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) streams, err := km.describeStreams(nil, nil) if err != nil { @@ -209,7 +212,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - km.awsManager.GetCollector().CollectFinish(km.Name) + km.awsManager.GetCollector().CollectFinish(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) return detectedStreams, nil } diff --git a/collector/aws/resources/lambda.go b/collector/aws/resources/lambda.go index e22f96e5..4de76190 100644 --- a/collector/aws/resources/lambda.go +++ b/collector/aws/resources/lambda.go @@ -71,7 +71,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err "resource": "lambda", }).Info("starting to analyze resource") - lm.awsManager.GetCollector().CollectStart(lm.Name) + lm.awsManager.GetCollector().CollectStart(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) detected := []DetectedAWSLambda{} functions, err := lm.describe(nil, nil) @@ -166,7 +169,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err } } - lm.awsManager.GetCollector().CollectFinish(lm.Name) + lm.awsManager.GetCollector().CollectFinish(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index c15d1fc8..ded329f9 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -75,7 +75,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "natgateway", }).Info("analyzing resource") - ngw.awsManager.GetCollector().CollectStart(ngw.Name) + ngw.awsManager.GetCollector().CollectStart(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) DetectedNATGateways := []DetectedNATGateway{} @@ -195,7 +198,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } - ngw.awsManager.GetCollector().CollectFinish(ngw.Name) + ngw.awsManager.GetCollector().CollectFinish(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) return DetectedNATGateways, nil } diff --git a/collector/aws/resources/neptune.go b/collector/aws/resources/neptune.go index 7092eee0..8a46eebc 100644 --- a/collector/aws/resources/neptune.go +++ b/collector/aws/resources/neptune.go @@ -75,7 +75,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "neptune", }).Info("starting to analyze resource") - np.awsManager.GetCollector().CollectStart(np.Name) + np.awsManager.GetCollector().CollectStart(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) detected := []DetectedAWSNeptune{} instances, err := np.describeInstances(nil, nil) @@ -178,7 +181,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - np.awsManager.GetCollector().CollectFinish(np.Name) + np.awsManager.GetCollector().CollectFinish(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index 6c9ee1a4..179bebfa 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -87,7 +87,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "rds", }).Info("starting to analyze resource") - r.awsManager.GetCollector().CollectStart(r.Name) + r.awsManager.GetCollector().CollectStart(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) detected := []DetectedAWSRDS{} @@ -224,7 +227,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) } - r.awsManager.GetCollector().CollectFinish(r.Name) + r.awsManager.GetCollector().CollectFinish(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index dc709e52..4dc3d7e0 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -74,7 +74,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "redshift", }).Info("analyzing resource") - rdm.awsManager.GetCollector().CollectStart(rdm.Name) + rdm.awsManager.GetCollector().CollectStart(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) detectedredshiftClusters := []DetectedRedShift{} @@ -176,7 +179,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } - rdm.awsManager.GetCollector().CollectFinish(rdm.Name) + rdm.awsManager.GetCollector().CollectFinish(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) return detectedredshiftClusters, nil } diff --git a/collector/collector.go b/collector/collector.go index b53b48c5..f3631f42 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -25,8 +25,8 @@ const ( // CollectorDescriber describe the collector functions type CollectorDescriber interface { AddResource(data EventCollector) - CollectStart(resourceName ResourceIdentifier) - CollectFinish(resourceName ResourceIdentifier) + CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) + CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) CollectError(resourceName ResourceIdentifier, err error) GetCollectorEvent() []EventCollector } @@ -98,21 +98,23 @@ func (cm *CollectorManager) AddResource(data EventCollector) { } // CollectStart add `fetch` event to collector by given resource name -func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFetch, + Status: EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } // CollectFinish add `finish` event to collector by given resource name -func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFinish, + Status: EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } diff --git a/collector/structs.go b/collector/structs.go index 58119e3b..9dc29506 100644 --- a/collector/structs.go +++ b/collector/structs.go @@ -23,8 +23,9 @@ const ( // EventStatusData descrive the struct of the resource statuses type EventStatusData struct { - Status EventStatus - ErrorMessage string + Status EventStatus + ErrorMessage string + AccountInformation string } // PriceDetectedFields describe the pricing field diff --git a/interpolation/interpolation.go b/interpolation/interpolation.go index f7268f17..8c936354 100644 --- a/interpolation/interpolation.go +++ b/interpolation/interpolation.go @@ -59,3 +59,12 @@ func ExtractExecutionName(executionId string) (string, error) { return executionArr[0], nil } + +// ExtractAccountInformation will return Account Name and Account ID +func ExtractAccountInformation(account string) (string, string, error) { + info := strings.Split(account, "_") + if len(info) != 2 { + return "", "", errors.New("unexpected account format") + } + return info[0], info[1], nil +} From 6cd0a85a4a6e893e54c73c3ff99c984ad34dbde3 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Sat, 5 Jun 2021 19:11:34 +0200 Subject: [PATCH 05/24] #SOS-23 Use querylimit in aggregation + add error handling for type assertion --- api/storage/elasticsearch/elasticsearch.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 9dcee359..836840ed 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -312,7 +312,8 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { accounts := []storage.Accounts{} - searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation().Field("Data.AccountInformation.keyword")). + searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation(). + Field("Data.AccountInformation.keyword").Size(querylimit)). Do(context.Background()) if err != nil { @@ -327,7 +328,11 @@ func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error } for _, accountsBucket := range resp.Buckets { - account := accountsBucket.Key.(string) + account, ok := accountsBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } name, id, err := interpolation.ExtractAccountInformation(account) if err != nil { log.WithError(err).WithField("account", account).Error("could not extract account information") From f8e931b57399d3da11fd686e7b7abccb24b3c4b9 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 13:53:06 +0200 Subject: [PATCH 06/24] #SOS-23 Fix existing tests + code format --- api/testutils/storage.go | 14 ++++++++++++++ collector/collector_test.go | 10 ++++++++-- collector/testutils/collector.go | 10 ++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 7a1a36a2..5100943c 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -65,6 +65,20 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro return response, nil } +func (ms *MockStorage) GetAccounts(querylimit int) ([]storage.Accounts, error) { + response := []storage.Accounts{ + { + ID: "1234567890", + Name: "Test1", + }, + { + ID: "1234567891", + Name: "Test2", + }, + } + return response, nil +} + func (ms *MockStorage) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { var response []map[string]interface{} diff --git a/collector/collector_test.go b/collector/collector_test.go index 55224438..88931e85 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -78,7 +78,10 @@ func TestAddEvent(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", Data: "test data", @@ -122,7 +125,10 @@ func TestAddEventServerUnavailable(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", diff --git a/collector/testutils/collector.go b/collector/testutils/collector.go index 0a41c9bb..81b7a9cc 100644 --- a/collector/testutils/collector.go +++ b/collector/testutils/collector.go @@ -23,20 +23,22 @@ func (mc *MockCollector) GetCollectorEvent() []collector.EventCollector { return events } -func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFetch, + Status: collector.EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } -func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFinish, + Status: collector.EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) From 259ccf7368fac954a659e97640deedb801d9b83b Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 15:14:22 +0200 Subject: [PATCH 07/24] #SOS-23 Modify GetAccounts for specific execution ID --- api/route.go | 5 ++++- api/server.go | 2 +- api/storage/elasticsearch/elasticsearch.go | 7 ++++--- api/storage/structs.go | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/api/route.go b/api/route.go index 73357bc1..756c2f32 100644 --- a/api/route.go +++ b/api/route.go @@ -57,7 +57,10 @@ func (server *Server) GetExecutions(resp http.ResponseWriter, req *http.Request) func (server *Server) GetAccounts(resp http.ResponseWriter, req *http.Request) { queryLimit, _ := strconv.Atoi(httpparameters.QueryParamWithDefault(req, "querylimit", storage.GetExecutionsQueryLimit)) - accounts, err := server.storage.GetAccounts(queryLimit) + params := mux.Vars(req) + executionID := params["executionID"] + accounts, err := server.storage.GetAccounts(executionID, queryLimit) + if err != nil { server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) return diff --git a/api/server.go b/api/server.go index 10ad2498..357c9c69 100644 --- a/api/server.go +++ b/api/server.go @@ -81,7 +81,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") - server.router.HandleFunc("/api/v1/accounts", server.GetAccounts).Methods(("GET")) + server.router.HandleFunc("/api/v1/accounts/{executionID}", server.GetAccounts).Methods(("GET")) server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 836840ed..0fd4403b 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -309,11 +309,12 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e return executions, nil } -func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { +func (sm *StorageManager) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { accounts := []storage.Accounts{} - searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation(). - Field("Data.AccountInformation.keyword").Size(querylimit)). + searchResult, err := sm.client.Search().Query(elastic.NewMatchQuery("ExecutionID", executionID)). + Aggregation("Accounts", elastic.NewTermsAggregation(). + Field("Data.AccountInformation.keyword").Size(querylimit)). Do(context.Background()) if err != nil { diff --git a/api/storage/structs.go b/api/storage/structs.go index 4d211301..a2bef4d6 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -13,7 +13,7 @@ type StorageDescriber interface { Save(data string) bool GetSummary(executionID string, filters map[string]string) (map[string]CollectorsSummary, error) GetExecutions(querylimit int) ([]Executions, error) - GetAccounts(querylimit int) ([]Accounts, error) + GetAccounts(executionID string, querylimit int) ([]Accounts, error) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) GetResourceTrends(resourceType string, filters map[string]string, limit int) ([]ExecutionCost, error) GetExecutionTags(executionID string) (map[string][]string, error) From 45db9b4d32310cea23153764872712d5ffffb4bf Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 15:30:04 +0200 Subject: [PATCH 08/24] #SOS-23 Fit MockStorage to modified interface StorageDescriber --- api/testutils/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 5100943c..7293503a 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -65,7 +65,7 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro return response, nil } -func (ms *MockStorage) GetAccounts(querylimit int) ([]storage.Accounts, error) { +func (ms *MockStorage) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { response := []storage.Accounts{ { ID: "1234567890", From 2f3c15cd9c083168ee35cdd2ef48eb2b1f2f2f04 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 18:31:03 +0200 Subject: [PATCH 09/24] #SOS-23 Add tests in interpolation_test.go and server_test.go --- api/server_test.go | 49 +++++++++++++++++++++++++++++ api/testutils/storage.go | 3 ++ interpolation/interpolation_test.go | 25 +++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/api/server_test.go b/api/server_test.go index 7ce3ccf0..a7cad068 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -235,8 +235,57 @@ func TestGetExecutions(t *testing.T) { }) } +} + +func TestGetAccounts(t *testing.T) { + ms, _ := MockServer() + ms.BindEndpoints() + ms.Serve() + testCases := []struct { + endpoint string + expectedStatusCode int + Count int + }{ + {"/api/v1/accounts", http.StatusNotFound, 0}, + {"/api/v1/accounts/1", http.StatusOK, 2}, + {"/api/v1/accounts/err", http.StatusInternalServerError, 0}, + } + + for _, test := range testCases { + t.Run(test.endpoint, func(t *testing.T) { + + rr := httptest.NewRecorder() + req, err := http.NewRequest("GET", test.endpoint, nil) + if err != nil { + t.Fatal(err) + } + ms.Router().ServeHTTP(rr, req) + if rr.Code != test.expectedStatusCode { + t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) + } + + if test.expectedStatusCode == http.StatusOK { + body, err := ioutil.ReadAll(rr.Body) + if err != nil { + t.Fatal(err) + } + + var accountsData []storage.Accounts + + err = json.Unmarshal(body, &accountsData) + if err != nil { + t.Fatalf("Could not parse http response") + } + + if len(accountsData) != test.Count { + t.Fatalf("unexpected resources summary response, got %d expected %d", len(accountsData), test.Count) + } + } + }) + } } + func TestSave(t *testing.T) { ms, mockStorage := MockServer() ms.BindEndpoints() diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 7293503a..cc3034b5 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -66,6 +66,9 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro } func (ms *MockStorage) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { + if executionID == "err" { + return nil, errors.New("error") + } response := []storage.Accounts{ { ID: "1234567890", diff --git a/interpolation/interpolation_test.go b/interpolation/interpolation_test.go index 51e5d4db..47612441 100644 --- a/interpolation/interpolation_test.go +++ b/interpolation/interpolation_test.go @@ -71,3 +71,28 @@ func TestExtractExecutionName(t *testing.T) { t.Errorf("extractedExecutionName %s is not equal to expected timestamp %s", extractedExecutionName, index_prefix) } } + +func TestExtractAccountInformation(t *testing.T) { + const name, id = "Test", "1234567890" + //test for right input + accountInfo := fmt.Sprintf("%s_%s", name, id) + extractedName, extractedId, err := interpolation.ExtractAccountInformation(accountInfo) + if err != nil { + t.Fatalf("error occured while running extractExecutionName e: %s\n", err) + } + + if extractedName != name { + t.Errorf("extractedName %s is not equal to expected name %s", extractedName, name) + } + + if extractedId != id { + t.Errorf("extractedId %s is not equal to expected id %s", extractedId, id) + } + + //test for wrong input + const wrong = "noUnderScore" + _, _, err = interpolation.ExtractAccountInformation(wrong) + if err == nil { + t.Errorf("function returns no error for input without underscore: %s", wrong) + } +} From 1fd641a5efbb858fe0a1b9f5cdef44cae82f3a28 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 9 Jun 2021 15:37:50 +0200 Subject: [PATCH 10/24] #SOS-20 * save arn in resourceID field + add values in test objects --- collector/aws/resources/apigateway.go | 11 ++++++- collector/aws/resources/ec2.go | 11 ++++++- collector/aws/resources/ec2volumes.go | 11 ++++++- collector/aws/resources/elasticache.go | 11 ++++++- collector/aws/resources/elasticips.go | 34 ++++++++++++++-------- collector/aws/resources/elasticips_test.go | 7 +++-- collector/aws/resources/elb.go | 11 ++++++- collector/aws/resources/elbv2.go | 2 +- collector/aws/resources/iam.go | 2 ++ collector/aws/resources/iam_test.go | 14 +++++++-- collector/aws/resources/kinesis.go | 2 +- collector/aws/resources/natgateway.go | 11 ++++++- collector/aws/resources/redshift.go | 11 ++++++- 13 files changed, 112 insertions(+), 26 deletions(-) diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index 118e5c65..ea79eaec 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -144,10 +145,18 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:apigateway:" + ag.awsManager.GetRegion() + "::/restapis/" + *api.Id + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + detect := DetectedAPIGateway{ Region: ag.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *api.Id, + ResourceID: Arn, Name: *api.Name, LaunchTime: *api.CreatedDate, Tag: tagsData, diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 52fe9f70..e4ba0d13 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "strings" "time" @@ -156,13 +157,21 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:ec2:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":instance/" + *instance.InstanceId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + ec2 := DetectedEC2{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, Name: name, InstanceType: *instance.InstanceType, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.InstanceId, + ResourceID: Arn, LaunchTime: *instance.LaunchTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index b4a5ccc0..cdf1908a 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -119,11 +120,19 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ev.awsManager.GetRegion() + ":" + *ev.awsManager.GetAccountIdentity().Account + ":volume/" + *vol.VolumeId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + volumeSize := *vol.Size dEBS := DetectedAWSEC2Volume{ Region: ev.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *vol.VolumeId, + ResourceID: Arn, Type: *vol.VolumeType, Size: volumeSize, PricePerMonth: ev.getCalculatedPrice(vol, price), diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index 96c9c4c7..06f42fe8 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -154,6 +155,14 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:elasticache:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":cluster:" + *instance.CacheClusterId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + es := DetectedElasticache{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, @@ -162,7 +171,7 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} CacheNodes: len(instance.CacheNodes), PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *instance.CacheClusterCreateTime, - ResourceID: *instance.CacheClusterId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index 27336e3c..dbc7d8a0 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -29,12 +30,10 @@ type ElasticIPManager struct { // DetectedElasticIP defines the detected AWS elastic ip type DetectedElasticIP struct { - Region string - Metric string - IP string - PricePerHour float64 - PricePerMonth float64 - Tag map[string]string + Region string + Metric string + IP string + collector.PriceDetectedFields collector.AccountSpecifiedFields } @@ -112,13 +111,24 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ei.awsManager.GetRegion() + ":" + *ei.awsManager.GetAccountIdentity().Account + ":elastic-ip/" + *ip.AllocationId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + eIP := DetectedElasticIP{ - Region: ei.awsManager.GetRegion(), - Metric: metric.Description, - IP: *ip.PublicIp, - PricePerHour: price, - PricePerMonth: price * collector.TotalMonthHours, - Tag: tagsData, + Region: ei.awsManager.GetRegion(), + Metric: metric.Description, + IP: *ip.PublicIp, + PriceDetectedFields: collector.PriceDetectedFields{ + ResourceID: Arn, + PricePerHour: price, + PricePerMonth: price * collector.TotalMonthHours, + Tag: tagsData, + }, AccountSpecifiedFields: collector.AccountSpecifiedFields{ AccountID: *ei.awsManager.GetAccountIdentity().Account, AccountName: ei.awsManager.GetAccountName(), diff --git a/collector/aws/resources/elasticips_test.go b/collector/aws/resources/elasticips_test.go index cce26355..56e14de5 100644 --- a/collector/aws/resources/elasticips_test.go +++ b/collector/aws/resources/elasticips_test.go @@ -21,12 +21,15 @@ var defaultAddressesMock = ec2.DescribeAddressesOutput{ AssociationId: awsClient.String("foo-00000"), InstanceId: awsClient.String("i-00000"), NetworkInterfaceId: awsClient.String("00000"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.81"), + PublicIp: awsClient.String("80.80.80.81"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.82"), + PublicIp: awsClient.String("80.80.80.82"), + AllocationId: awsClient.String("aws_eip.example.id"), }, }, } diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 14b77d1c..e64c20c8 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -170,11 +171,19 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:elasticloadbalancing:" + el.awsManager.GetRegion() + ":" + *el.awsManager.GetAccountIdentity().Account + ":loadbalancer/" + *instance.LoadBalancerName + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + elb := DetectedELB{ Region: el.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: Arn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index 4bc5e2b4..ec6eac89 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -217,7 +217,7 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro Metric: metric.Description, Type: *instance.Type, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: *instance.LoadBalancerArn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index dcd12e66..ad5ac99d 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -34,6 +34,7 @@ type DetectedAWSLastActivity struct { AccessKey string LastUsedDate time.Time LastActivity string + ResourceID string collector.AccountSpecifiedFields } @@ -137,6 +138,7 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) AccessKey: *accessKeyData.AccessKeyId, LastUsedDate: lastUsedDate, LastActivity: lastActivity, + ResourceID: *user.Arn, AccountSpecifiedFields: collector.AccountSpecifiedFields{ AccountID: *im.awsManager.GetAccountIdentity().Account, AccountName: im.awsManager.GetAccountName(), diff --git a/collector/aws/resources/iam_test.go b/collector/aws/resources/iam_test.go index cb6a6813..c1616e6b 100644 --- a/collector/aws/resources/iam_test.go +++ b/collector/aws/resources/iam_test.go @@ -16,9 +16,17 @@ import ( var defaultUsersMock = iam.ListUsersOutput{ Users: []*iam.User{ - {UserName: awsClient.String("foo")}, - {UserName: awsClient.String("foo2")}, - {UserName: awsClient.String("test")}, + { + UserName: awsClient.String("foo"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo")}, + { + UserName: awsClient.String("foo2"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo2"), + }, + { + UserName: awsClient.String("test"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/test"), + }, }, } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index b5471346..16c6e571 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -191,7 +191,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Region: km.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *stream.StreamName, + ResourceID: *stream.StreamARN, LaunchTime: *stream.StreamCreationTimestamp, PricePerHour: totalShardsPerHourPrice, PricePerMonth: totalShardsPerHourPrice * collector.TotalMonthHours, diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index ded329f9..52ceac21 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -170,6 +171,14 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:ec2:" + ngw.awsManager.GetRegion() + ":" + *ngw.awsManager.GetAccountIdentity().Account + ":natgateway/" + *natgateway.NatGatewayId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + natGateway := DetectedNATGateway{ Region: ngw.awsManager.GetRegion(), Metric: metric.Description, @@ -177,7 +186,7 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} VPCID: *natgateway.VpcId, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *natgateway.CreateTime, - ResourceID: *natgateway.NatGatewayId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index 4dc3d7e0..e791fac3 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -151,6 +152,14 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:redshift:" + rdm.awsManager.GetRegion() + ":" + *rdm.awsManager.GetAccountIdentity().Account + ":cluster:" + *cluster.ClusterIdentifier + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + redshift := DetectedRedShift{ Region: rdm.awsManager.GetRegion(), Metric: metric.Description, @@ -158,7 +167,7 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, NumberOfNodes: *cluster.NumberOfNodes, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *cluster.ClusterCreateTime, - ResourceID: *cluster.ClusterIdentifier, + ResourceID: Arn, PricePerHour: clusterPrice, PricePerMonth: clusterPrice * collector.TotalMonthHours, Tag: tagsData, From 9c157f214b6343853f2182a027415269082f5eb4 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Wed, 9 Jun 2021 19:46:46 +0200 Subject: [PATCH 11/24] #SOS-23 Add test for elasticsearch GetAccounts --- api/server_test.go | 2 +- .../elasticsearch/elasticsearch_test.go | 58 +++++++++++++++++++ .../accounts/aggregations/default.json | 12 ++++ interpolation/interpolation_test.go | 4 +- 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json diff --git a/api/server_test.go b/api/server_test.go index a7cad068..55192ec4 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -279,7 +279,7 @@ func TestGetAccounts(t *testing.T) { } if len(accountsData) != test.Count { - t.Fatalf("unexpected resources summary response, got %d expected %d", len(accountsData), test.Count) + t.Fatalf("unexpected accounts data response, got %d expected %d", len(accountsData), test.Count) } } }) diff --git a/api/storage/elasticsearch/elasticsearch_test.go b/api/storage/elasticsearch/elasticsearch_test.go index d3010bb3..5f78e112 100644 --- a/api/storage/elasticsearch/elasticsearch_test.go +++ b/api/storage/elasticsearch/elasticsearch_test.go @@ -217,6 +217,64 @@ func TestGetExecutions(t *testing.T) { } } +func TestGetAccounts(t *testing.T) { + + // each different queryLimit will result in a different elasticsearch response. + // 1 - returns valid account data response + // 2 - returns invalid aggregation term query response + // 4 - returns invalid statuscode response + testCases := []struct { + name string + queryLimit int + responseCount int + ErrorMessage error + }{ + {"valid response", 1, 2, nil}, + {"invalid terms", 2, 0, ErrAggregationTermNotFound}, + {"invalid es response", 3, 0, ErrInvalidQuery}, + } + + mockClient, config := testutils.NewESMock(prefixIndexName, true) + + mockClient.Router.HandleFunc("/_search", func(resp http.ResponseWriter, req *http.Request) { + switch testutils.GetPostParams(req) { + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":1}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "Accounts": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":2}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "invalid-key": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":3}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusBadRequest, elastic.SearchResult{Aggregations: map[string]json.RawMessage{}}) + default: + t.Fatalf("unexpected request params") + } + }) + + es, err := NewStorageManager(config) + if err != nil { + t.Fatalf("unexpected error, got %v expected nil", err) + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + + response, err := es.GetAccounts("1", test.queryLimit) + + if err != test.ErrorMessage { + t.Fatalf("unexpected error, got %v expected %v", err, test.ErrorMessage) + } + + if len(response) != test.responseCount { + t.Fatalf("handler query response: got %d want %d", len(response), test.responseCount) + } + + }) + } +} + func TestGetSummary(t *testing.T) { mockClient, config := testutils.NewESMock(prefixIndexName, true) diff --git a/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json new file mode 100644 index 00000000..ff15d0c8 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json @@ -0,0 +1,12 @@ +{ + "buckets" : [ + { + "key" : "Test_123456789", + "doc_count" : 66 + }, + { + "key" : "Test2_12345675", + "doc_count" : 6 + } + ] +} \ No newline at end of file diff --git a/interpolation/interpolation_test.go b/interpolation/interpolation_test.go index 47612441..221c64fd 100644 --- a/interpolation/interpolation_test.go +++ b/interpolation/interpolation_test.go @@ -78,7 +78,7 @@ func TestExtractAccountInformation(t *testing.T) { accountInfo := fmt.Sprintf("%s_%s", name, id) extractedName, extractedId, err := interpolation.ExtractAccountInformation(accountInfo) if err != nil { - t.Fatalf("error occured while running extractExecutionName e: %s\n", err) + t.Fatalf("error occured while running ExtractAccountInformation e: %s\n", err) } if extractedName != name { @@ -93,6 +93,6 @@ func TestExtractAccountInformation(t *testing.T) { const wrong = "noUnderScore" _, _, err = interpolation.ExtractAccountInformation(wrong) if err == nil { - t.Errorf("function returns no error for input without underscore: %s", wrong) + t.Errorf("ExtractAccountInformation returns no error for input without underscore: %s", wrong) } } From 0f68ad2b2b5a5f3ff952d6e24b71eea42d98a06f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Thu, 10 Jun 2021 15:57:59 +0200 Subject: [PATCH 12/24] #SOS-16 + add accounts to the provider store + Show Accounts in a chip-list like resources + add Accounts to filter --- ui/src/components/Dashboard/AccountsList.js | 82 +++++++++++++++++++++ ui/src/components/Dashboard/Index.js | 2 + ui/src/components/DataFactory.js | 21 ++++++ ui/src/reducers/accounts.reducer.js | 18 +++++ ui/src/reducers/index.js | 2 + ui/src/services/accs.service.js | 18 +++++ 6 files changed, 143 insertions(+) create mode 100644 ui/src/components/Dashboard/AccountsList.js create mode 100644 ui/src/reducers/accounts.reducer.js create mode 100644 ui/src/services/accs.service.js diff --git a/ui/src/components/Dashboard/AccountsList.js b/ui/src/components/Dashboard/AccountsList.js new file mode 100644 index 00000000..10bfa337 --- /dev/null +++ b/ui/src/components/Dashboard/AccountsList.js @@ -0,0 +1,82 @@ +import React, { Fragment } from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import colors from "./colors.json"; +import { makeStyles } from "@material-ui/core/styles"; +import { Box, Chip } from "@material-ui/core"; +import { setHistory } from "../../utils/History"; + +const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, + resource_chips: { + fontWeight: "bold", + fontFamily: "Arial !important", + margin: "5px", + borderRadius: "1px", + backgroundColor: "#ffffff", + borderLeft: "5px solid #ffffff", + fontSize: "14px", + }, +})); + +const AccountsList = ({ accounts, filters, addFilter }) => { + const classes = useStyles(); + + const accountsList = Object.values(accounts).map((account) => { + account.title = `${account.Name}(${account.ID})`; + return account; + }); + + const setSelectedAccount = (account) => { + const filter = { + title: `Account:${account.title}`, + id: `account:${account.ID}`, + type: "account", + }; + + addFilter(filter); + + setHistory({ + filters: filters, + }); + }; + + return ( + + {accountsList.length > 0 && ( + +

Accounts:

+ {accountsList.map((account, i) => ( + setSelectedAccount(account)} + ma={2} + label={account.title} + key={i} + /> + ))} + + )} + + ); +}; + +AccountsList.defaultProps = {}; +AccountsList.propTypes = { + accounts: PropTypes.object, + filters: PropTypes.array, + addFilter: PropTypes.func, +}; + +const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, + filters: state.filters.filters, +}); +const mapDispatchToProps = (dispatch) => ({ + addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AccountsList); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index a137f8e9..eb8a80f7 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -4,6 +4,7 @@ import { makeStyles } from "@material-ui/core/styles"; import { setHistory } from "../../utils/History"; import PropTypes from "prop-types"; +import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; @@ -71,6 +72,7 @@ const DashboardIndex = ({ + {currentResource ? : } diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index dd050fd1..ab440a11 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -5,6 +5,7 @@ import { ResourcesService } from "services/resources.service"; import { SettingsService } from "services/settings.service"; import { titleDirective } from "utils/Title"; import { getHistory, setHistory } from "../utils/History"; +import { AccsService } from "../services/accs.service"; let fetchTimeoutRequest = false; let fetchTableTimeoutRequest = false; @@ -38,6 +39,7 @@ const DataFacotry = ({ setCurrentExecution, currentResource, + setAccounts, setResources, setCurrentResourceData, setIsResourceListLoading, @@ -114,6 +116,7 @@ const DataFacotry = ({ clearTimeout(fetchTimeoutRequest); setIsResourceListLoading(true); await getResources(currentExecution, filters); + await getAccounts(currentExecution); setIsResourceListLoading(false); if (currentResource) { @@ -140,6 +143,20 @@ const DataFacotry = ({ setIsResourceTableLoading(false); }; + const getAccounts = async (currentExecution) => { + const AccountsArray = await AccsService.list(currentExecution).catch( + () => false + ); + + const accounts = {}; + AccountsArray.forEach((value) => { + accounts[value.ID] = value; + }); + + setAccounts(accounts); + return true; + }; + /** * Will fetch resource list from server * @param {string} currentExecution Current Selected Execution @@ -258,11 +275,13 @@ DataFacotry.propTypes = { setIsResourceListLoading: PropTypes.func, setIsResourceTableLoading: PropTypes.func, setIsScanning: PropTypes.func, + setAccounts: PropTypes.func, setResources: PropTypes.func, setCurrentResourceData: PropTypes.func, setCurrentExecution: PropTypes.func, currentResource: PropTypes.string, + accounts: PropTypes.object, resources: PropTypes.object, filters: PropTypes.array, currentExecution: PropTypes.string, @@ -273,6 +292,7 @@ DataFacotry.propTypes = { }; const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, resources: state.resources.resources, currentResource: state.resources.currentResource, currentExecution: state.executions.current, @@ -289,6 +309,7 @@ const mapDispatchToProps = (dispatch) => ({ setIsResourceTableLoading: (isLoading) => dispatch({ type: "IS_RESOURCE_TABLE_LOADING", isLoading }), setIsScanning: (isScanning) => dispatch({ type: "IS_SCANNING", isScanning }), + setAccounts: (data) => dispatch({ type: "ACCOUNT_LIST", data }), setResources: (data) => dispatch({ type: "RESOURCE_LIST", data }), setCurrentExecution: (id) => dispatch({ type: "EXECUTION_SELECTED", id }), setCurrentResourceData: (data) => diff --git a/ui/src/reducers/accounts.reducer.js b/ui/src/reducers/accounts.reducer.js new file mode 100644 index 00000000..b88b24ff --- /dev/null +++ b/ui/src/reducers/accounts.reducer.js @@ -0,0 +1,18 @@ +const initialState = { + accounts: {}, +}; + +/** + * @param {object} state module state + * @param {object} action to apply on state + * @returns {object} new copy of state + */ +export function accounts(state = initialState, action) { + switch (action.type) { + case "ACCOUNT_LIST": + state.accounts = action.data; + return { ...state }; + default: + return state; + } +} diff --git a/ui/src/reducers/index.js b/ui/src/reducers/index.js index f894e2dd..ad94d29c 100755 --- a/ui/src/reducers/index.js +++ b/ui/src/reducers/index.js @@ -1,11 +1,13 @@ import { combineReducers } from "redux"; import { connectRouter } from "connected-react-router"; +import { accounts } from "../reducers/accounts.reducer"; import { resources } from "../reducers/resources.reducer"; import { executions } from "../reducers/executions.reducer"; import { filters } from "../reducers/filters.reducer"; const rootReducer = (history) => combineReducers({ + accounts, resources, executions, filters, diff --git a/ui/src/services/accs.service.js b/ui/src/services/accs.service.js new file mode 100644 index 00000000..89666cd3 --- /dev/null +++ b/ui/src/services/accs.service.js @@ -0,0 +1,18 @@ +import { http } from "./request.service"; + +export const AccsService = { + list, +}; + +/** + * + * @param {string} executionId execution to query + */ +function list(executionId) { + return http + .send(`api/v1/accounts/${executionId}`, `get`) + .then(this.handleResponse) + .then((response) => { + return response; + }); +} From bf4c57d29327b0d56f30d54ca32935a3a9f455ba Mon Sep 17 00:00:00 2001 From: daniebrill Date: Fri, 11 Jun 2021 15:50:45 +0200 Subject: [PATCH 13/24] #SOS18 Add multiaccount selection functionality to tables --- ui/src/components/Dashboard/FilterBar.js | 7 +++++++ ui/src/services/resources.service.js | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ui/src/components/Dashboard/FilterBar.js b/ui/src/components/Dashboard/FilterBar.js index ca2f9137..f9822f20 100644 --- a/ui/src/components/Dashboard/FilterBar.js +++ b/ui/src/components/Dashboard/FilterBar.js @@ -137,6 +137,13 @@ const FilterBar = ({ type: "resource", }); resource = filterValue; + } else if (filterValue && filterKey === "account") { + filters.push({ + title: `Account:${filterValue}`, + id: filter, + value: filterValue, + type: "account", + }); } else if (filterValue) { const filterValues = filterValue.split(","); diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 878aa72b..8ab79481 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -14,12 +14,17 @@ export const ResourcesService = { const getTransformedFilters = (filters) => { const params = {}; filters.forEach((filter) => { - if (filter.id.substr(0, 8) === "resource") { + if (filter.type === "resource") { return; } const [key, value] = filter.id.split(":"); + let paramKey; + if (value && filter.type === "account") { + paramKey = `filter_Data.AccountID`; + } else { + paramKey = `filter_Data.Tag.${key}`; + } if (value) { - const paramKey = `filter_Data.Tag.${key}`; if (params[paramKey]) { params[paramKey] += `,${value}`; } else { From ecbe7ef1e81cbe7736079565565bd7409f5056da Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 11 Jun 2021 15:57:50 +0200 Subject: [PATCH 14/24] #SOS-16 add comments --- ui/src/components/Dashboard/AccountsList.js | 9 +++++++++ ui/src/components/DataFactory.js | 5 +++++ ui/src/services/resources.service.js | 1 + 3 files changed, 15 insertions(+) diff --git a/ui/src/components/Dashboard/AccountsList.js b/ui/src/components/Dashboard/AccountsList.js index 10bfa337..025b0170 100644 --- a/ui/src/components/Dashboard/AccountsList.js +++ b/ui/src/components/Dashboard/AccountsList.js @@ -21,6 +21,11 @@ const useStyles = makeStyles(() => ({ }, })); +/** + * @param {array} accounts Accounts List + * @param {array} filters Filters List + * @param {func} addFilter Add filter to filters list + */ const AccountsList = ({ accounts, filters, addFilter }) => { const classes = useStyles(); @@ -29,6 +34,10 @@ const AccountsList = ({ accounts, filters, addFilter }) => { return account; }); + /** + * + * @param {object} account add selected account + */ const setSelectedAccount = (account) => { const filter = { title: `Account:${account.title}`, diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index ab440a11..1e90ccf0 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -20,6 +20,7 @@ let lastFiltersSearched = "[]"; * @param {func} setCurrentExecution Update Current Execution * * @param {string} currentResource Current selected resource + * @param {func} setAccounts Update Accounts List * @param {func} setResources Update Resources List * @param {func} setCurrentResourceData Update current resource data * @param {func} setIsResourceListLoading update isLoading state for resources @@ -143,6 +144,10 @@ const DataFacotry = ({ setIsResourceTableLoading(false); }; + /** + * Will fetch account list from server + * @param {string} currentExecution current Selected Execution + */ const getAccounts = async (currentExecution) => { const AccountsArray = await AccsService.list(currentExecution).catch( () => false diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 878aa72b..c807f439 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -12,6 +12,7 @@ export const ResourcesService = { * @returns filters params for request */ const getTransformedFilters = (filters) => { + console.log(filters); const params = {}; filters.forEach((filter) => { if (filter.id.substr(0, 8) === "resource") { From 5ea12e5b721cfaf648c2d08dfe3093b5dd10cd8d Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 14 Jun 2021 16:47:54 +0200 Subject: [PATCH 15/24] #SOS-18 add spentAccounts in summary add things to success tests --- api/storage/elasticsearch/elasticsearch.go | 47 +++++++++++-------- .../elasticsearch/elasticsearch_test.go | 7 ++- .../summary/aggregations/default.json | 11 +++++ api/storage/structs.go | 13 ++--- 4 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 0fd4403b..471d1d55 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -194,7 +194,7 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri for resourceName, resourceData := range summary { filters["ResourceName"] = resourceName log.WithField("filters", filters).Debug("Going to get resources summary details with the following filters") - totalSpent, resourceCount, err := sm.getResourceSummaryDetails(executionID, filters) + totalSpent, resourceCount, spentAccounts, err := sm.getResourceSummaryDetails(executionID, filters) if err != nil { continue @@ -202,8 +202,8 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri newResourceData := resourceData newResourceData.TotalSpent = totalSpent newResourceData.ResourceCount = resourceCount + newResourceData.SpentAccounts = spentAccounts summary[resourceName] = newResourceData - } return summary, nil @@ -211,43 +211,52 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri } // getResourceSummaryDetails returns total resource spent and total resources detected -func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, error) { +func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, map[string]float64, error) { var totalSpent float64 var resourceCount int64 + var spentAccounts = make(map[string]float64) dynamicMatchQuery := sm.getDynamicMatchQuery(filters, "or") dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("ExecutionID", executionID)) dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("EventType", "resource_detected")) - searchResult, err := sm.client.Search(). + searchResultAccount, err := sm.client.Search(). Query(elastic.NewBoolQuery().Must(dynamicMatchQuery...)). - Aggregation("sum", elastic.NewSumAggregation().Field("Data.PricePerMonth")). + Aggregation("accounts", elastic.NewTermsAggregation().Field("Data.AccountID.keyword"). + SubAggregation("accountSum", elastic.NewSumAggregation().Field("Data.PricePerMonth"))). Size(0).Do(context.Background()) if err != nil { - log.WithError(err).WithFields(log.Fields{ - "filters": filters, - }).Error("error when trying to get summary details") + log.WithError(err).Error("error when trying to get executions collectors") + return totalSpent, resourceCount, spentAccounts, ErrInvalidQuery + } - return totalSpent, resourceCount, err + respAccount, ok := searchResultAccount.Aggregations.Terms("accounts") + if !ok { + log.Error("accounts field term does not exist") + return totalSpent, resourceCount, spentAccounts, ErrAggregationTermNotFound } - log.WithFields(log.Fields{ - "filters": filters, - "milliseconds": searchResult.TookInMillis, - }).Debug("get execution details") + for _, AccountIdBucket := range respAccount.Buckets { - resp, ok := searchResult.Aggregations.Terms("sum") - if ok { - if val, ok := resp.Aggregations["value"]; ok { + spent, ok := AccountIdBucket.Aggregations.Terms("accountSum") + if ok { + if val, ok := spent.Aggregations["value"]; ok { + accountID, ok := AccountIdBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } + spentAccounts[accountID], _ = strconv.ParseFloat(string(val), 64) - totalSpent, _ = strconv.ParseFloat(string(val), 64) - resourceCount = searchResult.Hits.TotalHits.Value + totalSpent += spentAccounts[accountID] + resourceCount += AccountIdBucket.DocCount + } } } - return totalSpent, resourceCount, nil + return totalSpent, resourceCount, spentAccounts, nil } // GetExecutions returns collector executions diff --git a/api/storage/elasticsearch/elasticsearch_test.go b/api/storage/elasticsearch/elasticsearch_test.go index 5f78e112..598e2e7d 100644 --- a/api/storage/elasticsearch/elasticsearch_test.go +++ b/api/storage/elasticsearch/elasticsearch_test.go @@ -283,7 +283,9 @@ func TestGetSummary(t *testing.T) { response := elastic.SearchResult{} - switch testutils.GetPostParams(req) { + var s = testutils.GetPostParams(req) + fmt.Println(s) + switch s { case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":0}`: response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":1}`: @@ -296,6 +298,9 @@ func TestGetSummary(t *testing.T) { case `{"aggregations":{"sum":{"sum":{"field":"Data.PricePerMonth"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: response.Aggregations = map[string]json.RawMessage{"sum": []byte(`{"value": 36.5}`)} response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} + case `{"aggregations":{"accounts":{"aggregations":{"accountSum":{"sum":{"field":"Data.PricePerMonth"}}},"terms":{"field":"Data.AccountID.keyword"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: + response.Aggregations = map[string]json.RawMessage{"accounts": testutils.LoadResponse("summary/aggregations/default")} + response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} default: t.Fatalf("unexpected request params") } diff --git a/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json new file mode 100644 index 00000000..082107a4 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json @@ -0,0 +1,11 @@ +{ + "buckets": [ + { + "key": "123456789012", + "doc_count": 1, + "accountSum": { + "value": 36.5 + } + } + ] +} \ No newline at end of file diff --git a/api/storage/structs.go b/api/storage/structs.go index a2bef4d6..0d453414 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -39,12 +39,13 @@ type Accounts struct { // CollectorsSummary defines unused resource summary type CollectorsSummary struct { - ResourceName string `json:"ResourceName"` - ResourceCount int64 `json:"ResourceCount"` - TotalSpent float64 `json:"TotalSpent"` - Status int `json:"Status"` - ErrorMessage string `json:"ErrorMessage"` - EventTime int64 `json:"-"` + ResourceName string `json:"ResourceName"` + ResourceCount int64 `json:"ResourceCount"` + TotalSpent float64 `json:"TotalSpent"` + Status int `json:"Status"` + ErrorMessage string `json:"ErrorMessage"` + EventTime int64 `json:"-"` + SpentAccounts map[string]float64 `json:"SpentAccounts"` } type SummaryData struct { From 2f995482ba90c3265fb3d1fe5dbb62c1f057db1f Mon Sep 17 00:00:00 2001 From: daniebrill Date: Mon, 14 Jun 2021 18:12:15 +0200 Subject: [PATCH 16/24] #SOS-18 Add multiaccount-selection functionality for RessourceCharts --- ui/src/components/Dashboard/FilterBar.js | 12 ++-- ui/src/components/Dashboard/Index.js | 3 +- ui/src/components/Dashboard/ResourcesChart.js | 56 +++++++++++++++---- .../components/Dashboard/ResourcesCharts.js | 36 ++++++++++++ 4 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 ui/src/components/Dashboard/ResourcesCharts.js diff --git a/ui/src/components/Dashboard/FilterBar.js b/ui/src/components/Dashboard/FilterBar.js index f9822f20..d029cb76 100644 --- a/ui/src/components/Dashboard/FilterBar.js +++ b/ui/src/components/Dashboard/FilterBar.js @@ -138,11 +138,13 @@ const FilterBar = ({ }); resource = filterValue; } else if (filterValue && filterKey === "account") { - filters.push({ - title: `Account:${filterValue}`, - id: filter, - value: filterValue, - type: "account", + const accounts = filterValue.split(","); + accounts.forEach((account) => { + filters.push({ + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }); }); } else if (filterValue) { const filterValues = filterValue.split(","); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index eb8a80f7..cb963357 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -9,6 +9,7 @@ import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; import ResourcesChart from "./ResourcesChart"; +import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; import ExecutionIndex from "../Executions/Index"; @@ -74,7 +75,7 @@ const DashboardIndex = ({ - {currentResource ? : } + {currentResource ? : } ); }; diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 42140213..fa1fa8ce 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -17,6 +17,9 @@ import { import ReportProblemIcon from "@material-ui/icons/ReportProblem"; const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, noDataTitle: { textAlign: "center", fontWeight: "bold", @@ -38,6 +41,8 @@ const useStyles = makeStyles(() => ({ * @param {bool} isResourceListLoading isLoading state for resources * @param {func} addFilter Add filter to filters list * @param {func} setResource Update Selected Resource} + * @param {string} account Account ID for account specific summary + * @param {object} accounts Accounts of current execution */ const ResourcesChart = ({ resources, @@ -45,12 +50,25 @@ const ResourcesChart = ({ isResourceListLoading, addFilter, setResource, + account, + accounts, }) => { const classes = useStyles(); const colorList = colors.map((color) => color.hex); - const sortedResources = Object.values(resources) - .filter((row) => row.TotalSpent > 0) - .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + let sortedResources; + if (account) { + sortedResources = Object.values(resources) + .filter( + (row) => row.SpentAccounts[account] && row.SpentAccounts[account] > 0 + ) + .sort((a, b) => + a.SpentAccounts[account] >= b.SpentAccounts[account] ? -1 : 1 + ); + } else { + sortedResources = Object.values(resources) + .filter((row) => row.TotalSpent > 0) + .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + } const chartOptions = { options: { @@ -148,12 +166,16 @@ const ResourcesChart = ({ */ sortedResources.forEach((resource) => { const title = titleDirective(resource.ResourceName); - const amount = MoneyDirective(resource.TotalSpent); + const amount = MoneyDirective( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); resource.title = `${title} (${amount})`; resource.display_title = `${title}`; chartOptions.options.xaxis.categories.push(resource.title); - chartOptions.series[0].data.push(resource.TotalSpent); + chartOptions.series[0].data.push( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); return resource; }); @@ -163,13 +185,20 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - + +

+ {account + ? `${accounts[account].Name} (${accounts[account].ID}):` + : "Summary:"} +

+ + )} {isResourceListLoading && ( @@ -194,12 +223,15 @@ ResourcesChart.propTypes = { isResourceListLoading: PropTypes.bool, addFilter: PropTypes.func, setResource: PropTypes.func, + account: PropTypes.string, + accounts: PropTypes.object, }; const mapStateToProps = (state) => ({ resources: state.resources.resources, isResourceListLoading: state.resources.isResourceListLoading, filters: state.filters.filters, + accounts: state.accounts.accounts, }); const mapDispatchToProps = (dispatch) => ({ diff --git a/ui/src/components/Dashboard/ResourcesCharts.js b/ui/src/components/Dashboard/ResourcesCharts.js new file mode 100644 index 00000000..1ba379d6 --- /dev/null +++ b/ui/src/components/Dashboard/ResourcesCharts.js @@ -0,0 +1,36 @@ +import React from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import { Fragment } from "react"; +import ResourcesChart from "./ResourcesChart"; + +/** + * @param {accounts} object Accounts of current execution + * @param {filters} array Filters list + */ +const ResourcesCharts = ({ accounts, filters }) => { + let selectedAccountIds = filters + .filter((filter) => filter.type === "account") + .map((filter) => filter.id.split(":")[1]); + if (selectedAccountIds.length === 0) { + selectedAccountIds = Object.keys(accounts); + } + let resourcesCharts = selectedAccountIds.map((accountID) => ( + + )); + resourcesCharts = [, ...resourcesCharts]; + return {resourcesCharts}; +}; + +ResourcesCharts.defaultProps = {}; +ResourcesCharts.propTypes = { + filters: PropTypes.array, + accounts: PropTypes.object, +}; + +const mapStateToProps = (state) => ({ + filters: state.filters.filters, + accounts: state.accounts.accounts, +}); + +export default connect(mapStateToProps)(ResourcesCharts); From 6ee0db86809f533d81aab1ad20fb04fd4fd1f6ab Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 30 Jun 2021 16:50:21 +0200 Subject: [PATCH 17/24] bugfix: Kinesis priceList error bugfix: ElasticIP priceList error bugfix: DocumentDB engineName error bugfix: DocumentDB priceList error --- collector/aws/pricing/pricing.go | 2 +- collector/aws/resources/docdb.go | 11 +++-- collector/aws/resources/elasticips.go | 63 ++++++++++++++++++--------- collector/aws/resources/kinesis.go | 15 +++++++ 4 files changed, 63 insertions(+), 28 deletions(-) diff --git a/collector/aws/pricing/pricing.go b/collector/aws/pricing/pricing.go index 60863a28..74d2127e 100644 --- a/collector/aws/pricing/pricing.go +++ b/collector/aws/pricing/pricing.go @@ -45,7 +45,7 @@ var regionsInfo = map[string]regionInfo{ "cn-north-1": {fullName: "China (Beijing)", prefix: ""}, "cn-northwest-1": {fullName: "China (Ningxia)", prefix: ""}, "eu-central-1": {fullName: "EU (Frankfurt)", prefix: "EUC1"}, - "eu-west-1": {fullName: "EU (Ireland)", prefix: "EUW1"}, + "eu-west-1": {fullName: "EU (Ireland)", prefix: "EU"}, "eu-west-2": {fullName: "EU (London)", prefix: "EUW2"}, "eu-west-3": {fullName: "EU (Paris)", prefix: "EUW3"}, "eu-south-1": {fullName: "EU (Milan)", prefix: "EUS1"}, diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index 1929dde3..a01fe263 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -209,6 +209,11 @@ func (dd *DocumentDBManager) getPricingFilterInput(instance *docdb.DBInstance) p Field: awsClient.String("instanceType"), Value: instance.DBInstanceClass, }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("productFamily"), + Value: awsClient.String("Database Instance"), + }, }, } } @@ -218,12 +223,6 @@ func (dd *DocumentDBManager) describeInstances(marker *string, instances []*docd input := &docdb.DescribeDBInstancesInput{ Marker: marker, - Filters: []*docdb.Filter{ - { - Name: awsClient.String("engine"), - Values: []*string{awsClient.String("docdb")}, - }, - }, } resp, err := dd.client.DescribeDBInstances(input) diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index dbc7d8a0..d02aa8c3 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "fmt" "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" @@ -79,14 +80,28 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, elasticIPs := []DetectedElasticIP{} - priceFIlters := ei.getPricingFilterInput() + pricingRegionPrefix, err := ei.awsManager.GetPricingClient().GetRegionPrefix(ei.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ei.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + ei.awsManager.GetCollector().CollectError(ei.Name, err) + return elasticIPs, err + } + + priceFilters := ei.getPricingFilterInput([]*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usagetype"), + Value: awsClient.String(fmt.Sprintf("%sElasticIP:IdleAddress", pricingRegionPrefix)), + }}) // Get elastic ip pricing - price, err := ei.awsManager.GetPricingClient().GetPrice(priceFIlters, ei.rateCode, ei.awsManager.GetRegion()) + price, err := ei.awsManager.GetPricingClient().GetPrice(priceFilters, ei.rateCode, ei.awsManager.GetRegion()) if err != nil { log.WithError(err).WithFields(log.Fields{ "rate_code": ei.rateCode, "region": ei.awsManager.GetRegion(), - "price_filters": priceFIlters, + "price_filters": priceFilters, }).Error("could not get elastic ip price") ei.awsManager.GetCollector().CollectError(ei.Name, err) @@ -155,27 +170,33 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } // getPricingFilterInput returns the elastic ip price filters. -func (ei *ElasticIPManager) getPricingFilterInput() pricing.GetProductsInput { +func (ei *ElasticIPManager) getPricingFilterInput(extraFilters []*pricing.Filter) pricing.GetProductsInput { + + filters := []*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("TermType"), + Value: awsClient.String("OnDemand"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("productFamily"), + Value: awsClient.String("IP Address"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("group"), + Value: awsClient.String("ElasticIP:Address"), + }, + } + + if extraFilters != nil { + filters = append(filters, extraFilters...) + } return pricing.GetProductsInput{ ServiceCode: &ei.servicePricingCode, - Filters: []*pricing.Filter{ - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("TermType"), - Value: awsClient.String("OnDemand"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("productFamily"), - Value: awsClient.String("IP Address"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("group"), - Value: awsClient.String("ElasticIP:Address"), - }, - }, + Filters: filters, } } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 16c6e571..c269e82c 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "fmt" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -98,6 +99,16 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er log.WithError(err).Error("Could not get shard price") return detectedStreams, err } + + pricingRegionPrefix, err := km.awsManager.GetPricingClient().GetRegionPrefix(km.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": km.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + km.awsManager.GetCollector().CollectError(km.Name, err) + return detectedStreams, err + } + // Get Price for extended Shard Hour retention extendedRetentionPrice, err := km.awsManager.GetPricingClient().GetPrice( km.getPricingFilterInput([]*pricing.Filter{ @@ -105,6 +116,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("group"), Value: awsClient.String("Addon shard hour"), + }, { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usageType"), + Value: awsClient.String(fmt.Sprintf("%sLoadBalancerUsage", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From c6b916e94b6fe6148b021380bb472cc7448b5c1f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 30 Jun 2021 17:20:26 +0200 Subject: [PATCH 18/24] bugfix: kinesis priceList Error --- collector/aws/resources/kinesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index c269e82c..6408f6fe 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -119,7 +119,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er }, { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usageType"), - Value: awsClient.String(fmt.Sprintf("%sLoadBalancerUsage", pricingRegionPrefix)), + Value: awsClient.String(fmt.Sprintf("%sStorage-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From 9242b0ab9a63dee20028b429462d13c62082a8e5 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 30 Jun 2021 17:36:54 +0200 Subject: [PATCH 19/24] bugfix: kinesis priceList error bugfix: elasticIP priceList error --- collector/aws/resources/elasticips.go | 5 ----- collector/aws/resources/kinesis.go | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index d02aa8c3..67546b36 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -183,11 +183,6 @@ func (ei *ElasticIPManager) getPricingFilterInput(extraFilters []*pricing.Filter Field: awsClient.String("productFamily"), Value: awsClient.String("IP Address"), }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("group"), - Value: awsClient.String("ElasticIP:Address"), - }, } if extraFilters != nil { diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 6408f6fe..28ff906e 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -119,7 +119,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er }, { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usageType"), - Value: awsClient.String(fmt.Sprintf("%sStorage-ShardHour", pricingRegionPrefix)), + Value: awsClient.String(fmt.Sprintf("%sExtended-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From db579e70143f993ffd99448d0d307a6bfb39af63 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Thu, 1 Jul 2021 15:55:22 +0200 Subject: [PATCH 20/24] bugfix: kinesis priceList error --- collector/aws/resources/kinesis.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 28ff906e..8cc388e5 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -87,6 +87,15 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er return detectedStreams, err } + pricingRegionPrefix, err := km.awsManager.GetPricingClient().GetRegionPrefix(km.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": km.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + km.awsManager.GetCollector().CollectError(km.Name, err) + return detectedStreams, err + } + // Get Price for regular Shard Hour shardPrice, err := km.awsManager.GetPricingClient().GetPrice(km.getPricingFilterInput( []*pricing.Filter{ @@ -94,21 +103,16 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("group"), Value: awsClient.String("Provisioned shard hour"), + }, { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usageType"), + Value: awsClient.String(fmt.Sprintf("%sExtended-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard price") return detectedStreams, err } - pricingRegionPrefix, err := km.awsManager.GetPricingClient().GetRegionPrefix(km.awsManager.GetRegion()) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "region": km.awsManager.GetRegion(), - }).Error("Could not get pricing region prefix") - km.awsManager.GetCollector().CollectError(km.Name, err) - return detectedStreams, err - } - // Get Price for extended Shard Hour retention extendedRetentionPrice, err := km.awsManager.GetPricingClient().GetPrice( km.getPricingFilterInput([]*pricing.Filter{ From 604e28a6e9d84d20ac83bc83007eb99ed99d7cbb Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Thu, 1 Jul 2021 16:06:06 +0200 Subject: [PATCH 21/24] bugfix: kinesis priceList Error --- collector/aws/resources/kinesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 8cc388e5..73eaebf4 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -106,7 +106,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er }, { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usageType"), - Value: awsClient.String(fmt.Sprintf("%sExtended-ShardHour", pricingRegionPrefix)), + Value: awsClient.String(fmt.Sprintf("%sStorage-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard price") From 4df494c82572853126641ba5ca3e394c62f97f7d Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 2 Jul 2021 16:01:02 +0200 Subject: [PATCH 22/24] fix warnings: unused statements --- ui/src/components/Dashboard/Index.js | 1 - ui/src/services/resources.service.js | 1 - 2 files changed, 2 deletions(-) diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index cb963357..5d48e9b8 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -8,7 +8,6 @@ import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; -import ResourcesChart from "./ResourcesChart"; import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index fa4604a6..8ab79481 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -12,7 +12,6 @@ export const ResourcesService = { * @returns filters params for request */ const getTransformedFilters = (filters) => { - console.log(filters); const params = {}; filters.forEach((filter) => { if (filter.type === "resource") { From ab6405e640d11bf5702ff7b4c4d13e6165864f54 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 23 Jul 2021 17:36:42 +0200 Subject: [PATCH 23/24] # SOS-57 remove account charts with not data found --- ui/src/components/Dashboard/ResourcesChart.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index fa1fa8ce..8a56a6a1 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -179,6 +179,10 @@ const ResourcesChart = ({ return resource; }); + if (account && !sortedResources.length && !isResourceListLoading) { + return ; + } + return ( From f8adc70e6e5ec579e78e6b84cdbc0437d2fe4f69 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 28 Jul 2021 15:32:55 +0200 Subject: [PATCH 24/24] set account to filters if an account specific Chart is clicked --- ui/src/components/Dashboard/ResourcesChart.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 8a56a6a1..92a76515 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -47,6 +47,7 @@ const useStyles = makeStyles(() => ({ const ResourcesChart = ({ resources, filters, + setFilters, isResourceListLoading, addFilter, setResource, @@ -81,6 +82,18 @@ const ResourcesChart = ({ const res = sortedResources; const selectedResource = res[dataPointIndex]; setSelectedResource(selectedResource); + if (account) { + const nfilters = filters.filter( + (filter) => filter.type !== "account" + ); + setFilters(nfilters); + const filter = { + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }; + addFilter(filter); + } }, }, }, @@ -224,6 +237,7 @@ ResourcesChart.defaultProps = {}; ResourcesChart.propTypes = { resources: PropTypes.object, filters: PropTypes.array, + setFilters: PropTypes.func, isResourceListLoading: PropTypes.bool, addFilter: PropTypes.func, setResource: PropTypes.func, @@ -239,6 +253,7 @@ const mapStateToProps = (state) => ({ }); const mapDispatchToProps = (dispatch) => ({ + setFilters: (data) => dispatch({ type: "SET_FILTERS", data }), addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), setResource: (data) => dispatch({ type: "SET_RESOURCE", data }), });