Merge branch 'alexjmoore-add_profile_support' into master

aws_list_all/__main__.py

@@ -75,6 +75,7 @@ def main():
     query.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel')
     query.add_argument('-d', '--directory', default='.', help='Directory to save result listings to')
     query.add_argument('-v', '--verbose', action='count', help='Print detailed info during run')
+    query.add_argument('-c', '--profile', help='Use a specific .aws/credentials profile.')
 
     # Once you have queried, show is the next most important command. So it comes second
     show = subparsers.add_parser(
@@ -149,7 +150,14 @@ def main():
             os.chdir(args.directory)
         increase_limit_nofiles()
         services = args.service or get_services()
-        do_query(services, args.region, args.operation, verbose=args.verbose or 0, parallel=args.parallel)
+        do_query(
+            services,
+            args.region,
+            args.operation,
+            verbose=args.verbose or 0,
+            parallel=args.parallel,
+            selected_profile=args.profile
+        )
     elif args.command == 'show':
         if args.listingfile:
             increase_limit_nofiles()

aws_list_all/client.py

@@ -3,8 +3,8 @@
 _CLIENTS = {}
 
 
-def get_client(service, region=None):
+def get_client(service, region=None, profile=None):
     """Return (cached) boto3 clients for this service and this region"""
     if (service, region) not in _CLIENTS:
-        _CLIENTS[(service, region)] = boto3.Session(region_name=region).client(service)
+        _CLIENTS[(service, region)] = boto3.Session(region_name=region, profile_name=profile).client(service)
     return _CLIENTS[(service, region)]

aws_list_all/introspection.py

@@ -320,10 +320,10 @@ def get_verbs(service):
     return set(re.sub('([A-Z])', '_\\1', x).split('_')[1] for x in client.meta.method_to_api_mapping.values())
 
 
-def get_listing_operations(service, region=None, selected_operations=()):
+def get_listing_operations(service, region=None, selected_operations=(), profile=None):
     """Return a list of API calls which (probably) list resources created by the user
     in the given service (in contrast to AWS-managed or default resources)"""
-    client = get_client(service, region)
+    client = get_client(service, region, profile)
     operations = []
     for operation in sorted(client.meta.service_model.operation_names):
         if not any(operation.startswith(prefix) for prefix in VERBS_LISTINGS):

aws_list_all/listing.py

@@ -66,9 +66,9 @@
 PARAMETERS.setdefault('cloudformation', {})['ListStacks'] = {'StackStatusFilter': ssf}
 
 
-def run_raw_listing_operation(service, region, operation):
+def run_raw_listing_operation(service, region, operation, profile):
     """Execute a given operation and return its raw result"""
-    client = get_client(service, region)
+    client = get_client(service, region, profile)
     api_to_method_mapping = dict((v, k) for k, v in client.meta.method_to_api_mapping.items())
     parameters = PARAMETERS.get(service, {}).get(operation, {})
     op_model = client.meta.service_model.operation_model(operation)
@@ -81,16 +81,18 @@ def run_raw_listing_operation(service, region, operation):
 
 class Listing(object):
     """Represents a listing operation on an AWS service and its result"""
-    def __init__(self, service, region, operation, response):
+    def __init__(self, service, region, operation, response, profile):
         self.service = service
         self.region = region
         self.operation = operation
         self.response = response
+        self.profile = profile
 
     def to_json(self):
         return {
             'service': self.service,
             'region': self.region,
+            'profile': self.profile,
             'operation': self.operation,
             'response': self.response,
         }
@@ -100,6 +102,7 @@ def from_json(cls, data):
         return cls(
             service=data.get('service'),
             region=data.get('region'),
+            profile=data.get('profile'),
             operation=data.get('operation'),
             response=data.get('response')
         )
@@ -120,18 +123,18 @@ def export_resources(self, filename):
             outfile.write(pprint.pformat(self.resources).encode('utf-8'))
 
     def __str__(self):
-        opdesc = '{} {} {}'.format(self.service, self.region, self.operation)
+        opdesc = '{} {} {} {}'.format(self.service, self.region, self.operation, self.profile)
         if len(self.resource_types) == 0 or self.resource_total_count == 0:
             return '{} (no resources found)'.format(opdesc)
         return opdesc + ', '.join('#{}: {}'.format(key, len(listing)) for key, listing in self.resources.items())
 
     @classmethod
-    def acquire(cls, service, region, operation):
+    def acquire(cls, service, region, operation, profile):
         """Acquire the given listing by making an AWS request"""
-        response = run_raw_listing_operation(service, region, operation)
+        response = run_raw_listing_operation(service, region, operation, profile)
         if response['ResponseMetadata']['HTTPStatusCode'] != 200:
             raise Exception('Bad AWS HTTP Status Code', response)
-        return cls(service, region, operation, response)
+        return cls(service, region, operation, response, profile)
 
     @property
     def resources(self):  # pylint:disable=too-many-branches
@@ -230,7 +233,7 @@ def resources(self):  # pylint:disable=too-many-branches
 
         # Special handling for service-level kms keys; derived from alias name.
         if self.service == 'kms' and self.operation == 'ListKeys':
-            list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases')
+            list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile)
             service_key_ids = [
                 k.get('TargetKeyId') for k in list_aliases.get('Aliases', [])
                 if k.get('AliasName').lower().startswith('alias/aws')
@@ -335,7 +338,7 @@ def resources(self):  # pylint:disable=too-many-branches
 
         # Filter default Internet Gateways
         if self.service == 'ec2' and self.operation == 'DescribeInternetGateways':
-            describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs')
+            describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile)
             vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])}
             internet_gateways = []
             for ig in response['InternetGateways']:

aws_list_all/query.py

@@ -195,19 +195,19 @@
 NOT_AVAILABLE_STRINGS = NOT_AVAILABLE_FOR_REGION_STRINGS + NOT_AVAILABLE_FOR_ACCOUNT_STRINGS
 
 
-def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32):
+def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None):
     """For the given services, execute all selected operations (default: all) in selected regions
     (default: all)"""
     to_run = []
     print('Building set of queries to execute...')
     for service in services:
         for region in get_regions_for_service(service, selected_regions):
-            for operation in get_listing_operations(service, region, selected_operations):
+            for operation in get_listing_operations(service, region, selected_operations, selected_profile):
                 if verbose > 0:
                     region_name = region or 'n/a'
                     print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation))
 
-                to_run.append([service, region, operation])
+                to_run.append([service, region, operation, selected_profile])
     shuffle(to_run)  # Distribute requests across endpoints
     results_by_type = defaultdict(list)
     print('...done. Executing queries...')
@@ -229,22 +229,22 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p
 def acquire_listing(verbose, what):
     """Given a service, region and operation execute the operation, serialize and save the result and
     return a tuple of strings describing the result."""
-    service, region, operation = what
+    service, region, operation, profile = what
     start_time = time()
     try:
         if verbose > 1:
             print(what, 'starting request...')
-        listing = Listing.acquire(service, region, operation)
+        listing = Listing.acquire(service, region, operation, profile)
         duration = time() - start_time
         if verbose > 1:
             print(what, '...request successful')
             print("timing [success]:", duration, what)
         if listing.resource_total_count > 0:
-            with open('{}_{}_{}.json'.format(service, operation, region), 'w') as jsonfile:
+            with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile:
                 json.dump(listing.to_json(), jsonfile, default=datetime.isoformat)
-            return (RESULT_SOMETHING, service, region, operation, ', '.join(listing.resource_types))
+            return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listing.resource_types))
         else:
-            return (RESULT_NOTHING, service, region, operation, ', '.join(listing.resource_types))
+            return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listing.resource_types))
     except Exception as exc:  # pylint:disable=broad-except
         duration = time() - start_time
         if verbose > 1:
@@ -266,7 +266,7 @@ def acquire_listing(verbose, what):
             if not_available_string in str(exc):
                 result_type = RESULT_NOTHING
 
-        return (result_type, service, region, operation, repr(exc))
+        return (result_type, service, region, operation, profile, repr(exc))
 
 
 def do_list_files(filenames, verbose=0):