Add support for specifying an External ID value in IAM role ARNs. (#366)

This is a breaking change as it requires the YAML config file to be
updated if you're currently using the `roleArns` field.

**Context**
IAM role delegation allows optionally setting an External ID string
value shared between parties. This is useful and recommended in order to
prevent the "confused deputy" problem when the account that can assume
the IAM role is outside your organisation.
When an External ID is setup for a role, users in the trusted account
must provide the exact identifier value to be able to assume the role.

More context here: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html

**Changes**
In this PR we change the config file format to allow specifying IAM
roles with both an ARN string and an External ID string. The credentials
handling code is updated to check for the presence of the External ID
value and pass it along in the assumeRole request, if present.

CHANGELOG.md

@@ -1,3 +1,22 @@
+# unreleased
+
+- *BREAKING CHANGE* Added support for specifying an External ID with IAM role Arns
+```yaml
+# Before
+discovery:
+  jobs:
+  - type: rds
+    roleArns:
+    - "arn:aws:iam::123456789012:role/Prometheus"
+# After
+discovery:
+  jobs:
+  - type: rds
+    roles:
+    - roleArn: "arn:aws:iam::123456789012:role/Prometheus"
+      externalId: "shared-external-identifier" # optional
+```
+
 # 0.27.0-alpha
 
 - Make exporter a library. (jeschkies)

README.md

@@ -117,7 +117,7 @@ exportedTagsOnMetrics:
     - type
 ```
 
-Note: Only [tagged resources](https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html) are discovered. 
+Note: Only [tagged resources](https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html) are discovered.
 
 ### Auto-discovery job
 
@@ -127,7 +127,7 @@ Note: Only [tagged resources](https://docs.aws.amazon.com/general/latest/gr/aws_
 | type                   | Cloudwatch service alias ("alb", "ec2", etc) or namespace name ("AWS/EC2", "AWS/S3", etc).                                                |
 | length (Default 120)   | How far back to request data for in seconds                                                              |
 | delay                  | If set it will request metrics up until `current_time - delay`                                           |
-| roleArns               | List of IAM roles to assume (optional)                                                                   |
+| roles                  | List of IAM roles to assume (optional)                                                                   |
 | searchTags             | List of Key/Value pairs to use for tag filtering (all must match), Value can be a regex.                 |
 | period                 | Statistic period in seconds (General Setting for all metrics in this job)                                |
 | addCloudwatchTimestamp | Export the metric with the original CloudWatch timestamp (General Setting for all metrics in this job)   |
@@ -164,7 +164,7 @@ general setting.  The currently inherited settings are period, and addCloudwatch
 | Key        | Description                                                |
 | ---------- | ---------------------------------------------------------- |
 | regions    | List of AWS regions                                        |
-| roleArns   | List of IAM roles to assume                                |
+| roles      | List of IAM roles to assume                                |
 | namespace  | CloudWatch namespace                                       |
 | name       | Must be set with multiple block definitions per namespace  |
 | customTags | Custom tags to be added as a list of Key/Value pairs       |
@@ -322,7 +322,7 @@ static:
         length: 300
 ```
 
-[Source: [config_test.yml](config_test.yml)]
+[Source: [config_test.yml](pkg/testdata/config_test.yml)]
 
 ## Metrics Examples
 
@@ -450,10 +450,10 @@ Multiple roleArns are useful, when you are monitoring multi-account setup, where
     - type: ecs-svc
       regions:
         - eu-north-1
-      roleArns:
-        - "arn:aws:iam::111111111111:role/prometheus" # newspaper
-        - "arn:aws:iam:2222222222222:role/prometheus" # radio
-        - "arn:aws:iam:3333333333333:role/prometheus" # television
+      roles:
+        - roleArn: "arn:aws:iam:1111111111111:role/prometheus" # newspaper
+        - roleArn: "arn:aws:iam:2222222222222:role/prometheus" # radio
+        - roleArn: "arn:aws:iam:3333333333333:role/prometheus" # television
       metrics:
         - name: MemoryReservation
           statistics:
@@ -464,6 +464,14 @@ Multiple roleArns are useful, when you are monitoring multi-account setup, where
           length: 600
 ```
 
+Additionally, if the IAM role you want to assume requires an [External ID](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html?icmpid=docs_iam_console) you can specify it this way:
+
+```yaml
+  roles:
+    - roleArn: "arn:aws:iam:1111111111111:role/prometheus"
+      externalId: "shared-external-identifier"
+```
+
 ### Requests concurrency
 The flags 'cloudwatch-concurrency' and 'tag-concurrency' define the number of concurrent request to cloudwatch metrics and tags. Their default value is 5.
 

pkg/abstract.go

@@ -19,28 +19,28 @@ func scrapeAwsData(config ScrapeConf, now time.Time, metricsPerQuery int, fips,
 	var wg sync.WaitGroup
 
 	for _, discoveryJob := range config.Discovery.Jobs {
-		for _, roleArn := range discoveryJob.RoleArns {
+		for _, role := range discoveryJob.Roles {
 			for _, region := range discoveryJob.Regions {
 				wg.Add(1)
-
-				go func(discoveryJob *Job, region string, roleArn string) {
+				go func(discoveryJob *Job, region string, role Role) {
 					defer wg.Done()
-					clientSts := createStsSession(roleArn)
+					clientSts := createStsSession(role)
 					result, err := clientSts.GetCallerIdentity(&sts.GetCallerIdentityInput{})
 					if err != nil {
-						log.Printf("Couldn't get account Id for role %s: %s\n", roleArn, err.Error())
+						log.Printf("Couldn't get account Id for role %s: %s\n", role.RoleArn, err.Error())
+
 					}
 					accountId := result.Account
 
 					clientCloudwatch := cloudwatchInterface{
-						client: createCloudwatchSession(&region, roleArn, fips),
+						client: createCloudwatchSession(&region, role, fips),
 					}
 
 					clientTag := tagsInterface{
-						client:           createTagSession(&region, roleArn, fips),
-						apiGatewayClient: createAPIGatewaySession(&region, roleArn, fips),
-						asgClient:        createASGSession(&region, roleArn, fips),
-						ec2Client:        createEC2Session(&region, roleArn, fips),
+						client:           createTagSession(&region, role, fips),
+						apiGatewayClient: createAPIGatewaySession(&region, role, fips),
+						asgClient:        createASGSession(&region, role, fips),
+						ec2Client:        createEC2Session(&region, role, fips),
 					}
 					var resources []*tagsData
 					var metrics []*cloudwatchData
@@ -49,35 +49,35 @@ func scrapeAwsData(config ScrapeConf, now time.Time, metricsPerQuery int, fips,
 					awsInfoData = append(awsInfoData, resources...)
 					cwData = append(cwData, metrics...)
 					mux.Unlock()
-				}(discoveryJob, region, roleArn)
+				}(discoveryJob, region, role)
 			}
 		}
 	}
 
 	for _, staticJob := range config.Static {
-		for _, roleArn := range staticJob.RoleArns {
+		for _, role := range staticJob.Roles {
 			for _, region := range staticJob.Regions {
 				wg.Add(1)
 
-				go func(staticJob *Static, region string, roleArn string) {
+				go func(staticJob *Static, region string, role Role) {
 					defer wg.Done()
-					clientSts := createStsSession(roleArn)
+					clientSts := createStsSession(role)
 					result, err := clientSts.GetCallerIdentity(&sts.GetCallerIdentityInput{})
 					if err != nil {
-						log.Printf("Couldn't get account Id for role %s: %s\n", roleArn, err.Error())
+						log.Printf("Couldn't get account Id for role %s: %s\n", role.RoleArn, err.Error())
 					}
 					accountId := result.Account
 
 					clientCloudwatch := cloudwatchInterface{
-						client: createCloudwatchSession(&region, roleArn, fips),
+						client: createCloudwatchSession(&region, role, fips),
 					}
 
 					metrics := scrapeStaticJob(staticJob, region, accountId, clientCloudwatch, cloudwatchSemaphore)
 
 					mux.Lock()
 					cwData = append(cwData, metrics...)
 					mux.Unlock()
-				}(staticJob, region, roleArn)
+				}(staticJob, region, role)
 			}
 		}
 	}

pkg/aws_cloudwatch.go

@@ -47,7 +47,7 @@ type cloudwatchData struct {
 
 var labelMap = make(map[string][]string)
 
-func createStsSession(roleArn string) *sts.STS {
+func createStsSession(role Role) *sts.STS {
 	sess := session.Must(session.NewSessionWithOptions(session.Options{
 		SharedConfigState: session.SharedConfigEnable,
 	}))
@@ -56,13 +56,17 @@ func createStsSession(roleArn string) *sts.STS {
 	if log.IsLevelEnabled(log.DebugLevel) {
 		config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
 	}
-	if roleArn != "" {
-		config.Credentials = stscreds.NewCredentials(sess, roleArn)
+	if role.RoleArn != "" {
+		config.Credentials = stscreds.NewCredentials(sess, role.RoleArn, func(p *stscreds.AssumeRoleProvider) {
+			if role.ExternalID != "" {
+				p.ExternalID = aws.String(role.ExternalID)
+			}
+		})
 	}
 	return sts.New(sess, config)
 }
 
-func createCloudwatchSession(region *string, roleArn string, fips bool) *cloudwatch.CloudWatch {
+func createCloudwatchSession(region *string, role Role, fips bool) *cloudwatch.CloudWatch {
 	sess := session.Must(session.NewSessionWithOptions(session.Options{
 		SharedConfigState: session.SharedConfigEnable,
 		Config:            aws.Config{Region: aws.String(*region)},
@@ -82,8 +86,12 @@ func createCloudwatchSession(region *string, roleArn string, fips bool) *cloudwa
 		config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
 	}
 
-	if roleArn != "" {
-		config.Credentials = stscreds.NewCredentials(sess, roleArn)
+	if role.RoleArn != "" {
+		config.Credentials = stscreds.NewCredentials(sess, role.RoleArn, func(p *stscreds.AssumeRoleProvider) {
+			if role.ExternalID != "" {
+				p.ExternalID = aws.String(role.ExternalID)
+			}
+		})
 	}
 
 	return cloudwatch.New(sess, config)

pkg/aws_tags.go

@@ -34,18 +34,22 @@ type tagsInterface struct {
 	ec2Client        ec2iface.EC2API
 }
 
-func createSession(roleArn string, config *aws.Config) *session.Session {
+func createSession(role Role, config *aws.Config) *session.Session {
 	sess, err := session.NewSession(config)
 	if err != nil {
 		log.Fatalf("Failed to create session due to %v", err)
 	}
-	if roleArn != "" {
-		config.Credentials = stscreds.NewCredentials(sess, roleArn)
+	if role.RoleArn != "" {
+		config.Credentials = stscreds.NewCredentials(sess, role.RoleArn, func(p *stscreds.AssumeRoleProvider) {
+			if role.ExternalID != "" {
+				p.ExternalID = aws.String(role.ExternalID)
+			}
+		})
 	}
 	return sess
 }
 
-func createTagSession(region *string, roleArn string, fips bool) *r.ResourceGroupsTaggingAPI {
+func createTagSession(region *string, role Role, fips bool) *r.ResourceGroupsTaggingAPI {
 	maxResourceGroupTaggingRetries := 5
 	config := &aws.Config{Region: region, MaxRetries: &maxResourceGroupTaggingRetries}
 	if fips {
@@ -54,10 +58,10 @@ func createTagSession(region *string, roleArn string, fips bool) *r.ResourceGrou
 		// endpoint := fmt.Sprintf("https://tagging-fips.%s.amazonaws.com", *region)
 		// config.Endpoint = aws.String(endpoint)
 	}
-	return r.New(createSession(roleArn, config), config)
+	return r.New(createSession(role, config), config)
 }
 
-func createASGSession(region *string, roleArn string, fips bool) autoscalingiface.AutoScalingAPI {
+func createASGSession(region *string, role Role, fips bool) autoscalingiface.AutoScalingAPI {
 	maxAutoScalingAPIRetries := 5
 	config := &aws.Config{Region: region, MaxRetries: &maxAutoScalingAPIRetries}
 	if fips {
@@ -66,36 +70,29 @@ func createASGSession(region *string, roleArn string, fips bool) autoscalingifac
 		// endpoint := fmt.Sprintf("https://autoscaling-plans-fips.%s.amazonaws.com", *region)
 		// config.Endpoint = aws.String(endpoint)
 	}
-	return autoscaling.New(createSession(roleArn, config), config)
+	return autoscaling.New(createSession(role, config), config)
 }
 
-func createEC2Session(region *string, roleArn string, fips bool) ec2iface.EC2API {
+func createEC2Session(region *string, role Role, fips bool) ec2iface.EC2API {
 	maxEC2APIRetries := 10
 	config := &aws.Config{Region: region, MaxRetries: &maxEC2APIRetries}
 	if fips {
 		// https://docs.aws.amazon.com/general/latest/gr/ec2-service.html
 		endpoint := fmt.Sprintf("https://ec2-fips.%s.amazonaws.com", *region)
 		config.Endpoint = aws.String(endpoint)
 	}
-	return ec2.New(createSession(roleArn, config), config)
+	return ec2.New(createSession(role, config), config)
 }
 
-func createAPIGatewaySession(region *string, roleArn string, fips bool) apigatewayiface.APIGatewayAPI {
+func createAPIGatewaySession(region *string, role Role, fips bool) apigatewayiface.APIGatewayAPI {
 	maxApiGatewaygAPIRetries := 5
 	config := &aws.Config{Region: region, MaxRetries: &maxApiGatewaygAPIRetries}
-	sess, err := session.NewSession(config)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if roleArn != "" {
-		config.Credentials = stscreds.NewCredentials(sess, roleArn)
-	}
 	if fips {
 		// https://docs.aws.amazon.com/general/latest/gr/apigateway.html
 		endpoint := fmt.Sprintf("https://apigateway-fips.%s.amazonaws.com", *region)
 		config.Endpoint = aws.String(endpoint)
 	}
-	return apigateway.New(sess, config)
+	return apigateway.New(createSession(role, config), config)
 }
 
 func (iface tagsInterface) get(job *Job, region string) (resources []*tagsData, err error) {