From a0477e14faf4c925a64d7322985e2a062650fd74 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 4 Sep 2015 12:22:43 +0100 Subject: [PATCH 01/12] security group for vpcs --- gonzo/clouds/compute.py | 129 ++++++++++++++++++++++++++++++++-------- gonzo/scripts/launch.py | 6 +- 2 files changed, 110 insertions(+), 25 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 2770777..5c7b465 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -99,7 +99,7 @@ def get_next_az(self, environment, server_type): def create_instance(self, image_name, name, owner, user_data=None, security_groups=None, size=None, key_name=None, - volume_size=None): + volume_size=None, subnet_id=None): instance_name = name.split('-') environment = instance_name[0] server_type = '-'.join(instance_name[1:-1]) @@ -126,8 +126,11 @@ def create_instance(self, image_name, name, owner, user_data=None, if security_groups is None: security_groups = [] - for security_group in security_groups: - self.create_if_not_exist_security_group(security_group) + for index, security_group in enumerate(security_groups): + self.create_if_not_exist_security_group(security_group, subnet_id) + security_groups[index] = self.get_security_group( + security_group, subnet_id + ) security_groups_for_launch = self.security_groups_for_launch( security_groups) @@ -217,28 +220,13 @@ def generate_instance_metadata(self, owner, environment, server_type): instance_metadata['server_type'] = server_type return instance_metadata - def create_if_not_exist_security_group(self, group_name): - - try: - desc = "Rules for {}".format(group_name) - self.compute_session.ex_create_security_group(group_name, desc) - - except Exception as exc: # libcloud doesn't raise anything better - if not ("exists" in str(exc)): - raise - - def get_security_group(self, group_name): - - for group in self.list_security_groups(): - if group_name == getattr(group, self.SECURITY_GROUP_IDENTIFIER): - return group - def list_security_groups(self): group_list_method = getattr(self.compute_session, self.SECURITY_GROUP_METHOD) return group_list_method() + @backend_for('ec2') class AWS(Cloud): TAG_KEY = 'tags' @@ -266,7 +254,16 @@ def _monkeypatch_instance(self, instance): instance.extra['gonzo_network_address'] = instance.extra['dns_name'] def security_groups_for_launch(self, security_group_names): - return security_group_names + vpc_id = security_group_names[0].extra['vpc_id'] + + if vpc_id: + identifier = 'id' + else: + identifier = 'name' + + return [ + getattr(group, identifier) for group in security_group_names + ] def create_volume(self, instance, vol_name, vol_size, vol_type='gp2'): @@ -278,6 +275,65 @@ def create_volume(self, instance, vol_name, ex_volume_type=vol_type, ) + def create_if_not_exist_security_group(self, group_name, subnet_id=None): + if subnet_id is not None: + subnet = self.compute_session.ex_list_subnets([subnet_id])[0] + subnet_vpc = subnet.extra['vpc_id'] + else: + subnet_vpc = None + + try: + desc = "Rules for {}".format(group_name) + self.compute_session.ex_create_security_group( + group_name, + desc, + subnet_vpc, + ) + except Exception as exc: # libcloud doesn't raise anything better + if not ("exists" in str(exc)): + raise + + def get_subnet(self, subnet_id): + for subnet in self.compute_session.ex_list_subnets(): + if subnet.id == subnet_id: + return subnet + raise LookupError("Subnet containting subnet: {} not found".format( + subnet_id)) + + def get_security_group(self, group_name, subnet_id=None): + filters = None + if subnet_id: + filters = {'vpc-id': self.get_vpc_for_subnet(subnet_id)} + + sec_groups = self.compute_session.ex_get_security_groups( + filters=filters) + + for sec_group in sec_groups: + if sec_group.name == group_name: + return sec_group + + def get_vpc_for_subnet(self, subnet_id): + try: + subnet = self.compute_session.ex_list_subnets([subnet_id]) + return subnet[0].extra['vpc_id'] + except IndexError: + raise LookupError("VPC for subnet not found") + + def _generate_instance_dict(self, **kwargs): + + vpc_exclude = ['ex_security_groups', 'location'] + classic_exclude = ['ex_subnet', 'ex_security_group_ids'] + + instance_dict = kwargs + if instance_dict['ex_subnet']: + for exclude in vpc_exclude: + instance_dict.pop(exclude, None) + else: + for exclude in classic_exclude: + instance_dict.pop(exclude, None) + + return instance_dict + @backend_for(ComputeProvider.OPENSTACK) class Openstack(Cloud): @@ -323,13 +379,38 @@ def _monkeypatch_instance(self, instance): instance.extra['gonzo_network_address'] = instance.private_ips[0] def security_groups_for_launch(self, security_group_names): - return [ - self.get_security_group(name) - for name in security_group_names - ] + return security_group_names def create_volume(self, instance, vol_name, vol_size, vol_type='gp2'): return self.compute_session.create_volume( name=vol_name, size=vol_size, ) + + def create_if_not_exist_security_group(self, group_name, subnet_id=None): + try: + desc = "Rules for {}".format(group_name) + self.compute_session.ex_create_security_group(group_name, desc) + + except Exception as exc: # libcloud doesn't raise anything better + if not ("exists" in str(exc)): + raise + + def get_security_group(self, group_name, subnet_id=None): + if subnet_id: + raise Exception("VPC not supported for Openstack") + + for group in self.list_security_groups(): + if group_name == getattr(group, self.SECURITY_GROUP_IDENTIFIER): + return group + + def _generate_instance_dict(self, **kwargs): + + vpc_exclude = ['ex_security_groups', 'location'] + + instance_dict = kwargs + if instance_dict['ex_subnet']: + for exclude in vpc_exclude: + instance_dict.pop(exclude, None) + + return instance_dict diff --git a/gonzo/scripts/launch.py b/gonzo/scripts/launch.py index 833807f..501e7e8 100755 --- a/gonzo/scripts/launch.py +++ b/gonzo/scripts/launch.py @@ -104,7 +104,8 @@ def launch(args): security_groups=security_groups, owner=username, key_name=cloud_config.get('PUBLIC_KEY_NAME'), - volume_size=args.volume_size + volume_size=args.volume_size, + subnet_id=args.subnet_id, ) print "Instance created: {}.{}".format( @@ -161,6 +162,9 @@ def init_parser(parser): parser.add_argument( '--size', dest='size', # choices=config.CLOUD['SIZES'], help="Override instance size") + parser.add_argument( + '--subnet-id', dest='subnet_id', + help="VPC Subnet ID") parser.add_argument( '--user-data-uri', dest='user_data_uri', help=user_data_help) From 01ffa68c86c4be10e6b1ae9cd5696a36fb155f88 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 4 Sep 2015 12:23:18 +0100 Subject: [PATCH 02/12] vpc subnet settings --- gonzo/clouds/compute.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 5c7b465..7e47c29 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -135,8 +135,14 @@ def create_instance(self, image_name, name, owner, user_data=None, security_groups_for_launch = self.security_groups_for_launch( security_groups) - # Availability Zone - az = self.get_next_az(environment, server_type) + + # VPC Specific + if subnet_id is not None: + az = self.get_next_az(environment, server_type) + subnet = self.get_subnet(subnet_id) + else: + az = None + subnet = None # Launch Instance instance = self.compute_session.create_node( From 41920a60222436a8c1a3fef136274644f1740148 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 4 Sep 2015 12:23:47 +0100 Subject: [PATCH 03/12] map instance launch options for different clouds --- gonzo/clouds/compute.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 7e47c29..4e7b886 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -145,16 +145,23 @@ def create_instance(self, image_name, name, owner, user_data=None, subnet = None # Launch Instance - instance = self.compute_session.create_node( + instance_dict = self._generate_instance_dict( name=name, image=image, size=size, location=az, ex_security_groups=security_groups_for_launch, + ex_security_group_ids=security_groups_for_launch, ex_metadata=tags, ex_userdata=user_data, ex_keyname=key_name, + ex_subnet=subnet ) + + instance = self.compute_session.create_node( + **instance_dict + ) + self.compute_session.wait_until_running([instance]) new_instance = self.get_instance_by_uuid(instance.uuid) From e15074f834be9e594b1838e26974e09c89fd2027 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 4 Sep 2015 12:24:07 +0100 Subject: [PATCH 04/12] bump libcloud version to 0.18.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5158cda..5a79f6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apache-libcloud==0.16.0 +apache-libcloud==0.18.0 jinja2==2.7.1 argcomplete==0.8.4 fabric==1.6 From 300bcdf4e69c411d860a8c5edc333e7dad01e520 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 4 Sep 2015 12:44:42 +0100 Subject: [PATCH 05/12] flake8 --- gonzo/clouds/compute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 4e7b886..f485934 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -135,7 +135,6 @@ def create_instance(self, image_name, name, owner, user_data=None, security_groups_for_launch = self.security_groups_for_launch( security_groups) - # VPC Specific if subnet_id is not None: az = self.get_next_az(environment, server_type) @@ -239,7 +238,6 @@ def list_security_groups(self): return group_list_method() - @backend_for('ec2') class AWS(Cloud): TAG_KEY = 'tags' From 298630efd9b2f394a61798270ea27fdc9e92424f Mon Sep 17 00:00:00 2001 From: Amit Date: Wed, 9 Sep 2015 17:57:13 +0100 Subject: [PATCH 06/12] dynamically determine dns record type --- gonzo/scripts/launch.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gonzo/scripts/launch.py b/gonzo/scripts/launch.py index 501e7e8..961e38f 100755 --- a/gonzo/scripts/launch.py +++ b/gonzo/scripts/launch.py @@ -113,9 +113,19 @@ def launch(args): cloud_config['DNS_ZONE'] ) + dns_record_type = 'A' + dns_value = instance.extra['gonzo_network_address'] + + if args.subnet_id: + if instance.extra.get('gonzo_network_address'): + dns_record_type = 'CNAME' + else: + dns_value = instance.private_ips[0] + + print dns_record_type dns.create_dns_record(instance.name, - instance.extra['gonzo_network_address'], - cloud_config['DNS_TYPE'], + dns_value, + dns_record_type, cloud_config['DNS_ZONE']) From e415fca6062fb24dc78b6bdf29a8b9a8b2656607 Mon Sep 17 00:00:00 2001 From: Amit Date: Wed, 9 Sep 2015 17:59:04 +0100 Subject: [PATCH 07/12] stray print --- gonzo/scripts/launch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gonzo/scripts/launch.py b/gonzo/scripts/launch.py index 961e38f..918c826 100755 --- a/gonzo/scripts/launch.py +++ b/gonzo/scripts/launch.py @@ -122,7 +122,6 @@ def launch(args): else: dns_value = instance.private_ips[0] - print dns_record_type dns.create_dns_record(instance.name, dns_value, dns_record_type, From c542ee258a9c41ff839791e3e3707aa120390636 Mon Sep 17 00:00:00 2001 From: Amit Date: Thu, 10 Sep 2015 14:27:39 +0100 Subject: [PATCH 08/12] change log --- CHANGES | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index a41158b..a315b7d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +Version 0.5.0 +------------- + +- Launch AWS instances into a VPC subnet +- VPC support for security groups +- DNS creation based on instance type + Version 0.4.2 ------------- diff --git a/setup.py b/setup.py index d6b2e7c..a4c32b3 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def parse_requirments(fn, dependency_links): setup( name='gonzo', packages=find_packages(exclude=['tests', 'tests.*']), - version='0.4.2', + version='0.5.0', author='onefinestay', author_email='engineering@onefinestay.com', url='https://github.com/onefinestay/gonzo', From 7423f2d49aa33f99baa9c33d910fbbae860b3053 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 22 Jan 2016 15:36:28 +0000 Subject: [PATCH 09/12] tidy up --- gonzo/clouds/compute.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index f485934..22cf48e 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -267,10 +267,7 @@ def _monkeypatch_instance(self, instance): def security_groups_for_launch(self, security_group_names): vpc_id = security_group_names[0].extra['vpc_id'] - if vpc_id: - identifier = 'id' - else: - identifier = 'name' + identifier = 'id' if vpc_id else 'name' return [ getattr(group, identifier) for group in security_group_names @@ -287,14 +284,14 @@ def create_volume(self, instance, vol_name, ) def create_if_not_exist_security_group(self, group_name, subnet_id=None): - if subnet_id is not None: + if subnet_id: subnet = self.compute_session.ex_list_subnets([subnet_id])[0] subnet_vpc = subnet.extra['vpc_id'] else: subnet_vpc = None + desc = "Rules for {}".format(group_name) try: - desc = "Rules for {}".format(group_name) self.compute_session.ex_create_security_group( group_name, desc, @@ -331,7 +328,6 @@ def get_vpc_for_subnet(self, subnet_id): raise LookupError("VPC for subnet not found") def _generate_instance_dict(self, **kwargs): - vpc_exclude = ['ex_security_groups', 'location'] classic_exclude = ['ex_subnet', 'ex_security_group_ids'] From 2bb5ab205c1e57ada50908d9b4bbc3a015a5d566 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 22 Jan 2016 15:50:36 +0000 Subject: [PATCH 10/12] availability zone comment --- gonzo/clouds/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 22cf48e..70c8862 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -135,13 +135,13 @@ def create_instance(self, image_name, name, owner, user_data=None, security_groups_for_launch = self.security_groups_for_launch( security_groups) - # VPC Specific - if subnet_id is not None: + # VPC / Availability Zone + az = None + subnet = None + + if subnet_id: az = self.get_next_az(environment, server_type) subnet = self.get_subnet(subnet_id) - else: - az = None - subnet = None # Launch Instance instance_dict = self._generate_instance_dict( From e354d7c0906ed37e598b76495e9069c0c7fb4f80 Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 22 Jan 2016 15:50:56 +0000 Subject: [PATCH 11/12] clearer naming --- gonzo/clouds/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index 70c8862..e78eb55 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -264,13 +264,13 @@ def _monkeypatch_instance(self, instance): instance.extra['gonzo_az'] = instance.extra['availability'] instance.extra['gonzo_network_address'] = instance.extra['dns_name'] - def security_groups_for_launch(self, security_group_names): - vpc_id = security_group_names[0].extra['vpc_id'] + def security_groups_for_launch(self, security_groups): + vpc_id = security_groups[0].extra['vpc_id'] identifier = 'id' if vpc_id else 'name' return [ - getattr(group, identifier) for group in security_group_names + getattr(group, identifier) for group in security_groups ] def create_volume(self, instance, vol_name, @@ -385,8 +385,8 @@ def _monkeypatch_instance(self, instance): instance.extra['gonzo_az'] = instance.extra['availability_zone'] instance.extra['gonzo_network_address'] = instance.private_ips[0] - def security_groups_for_launch(self, security_group_names): - return security_group_names + def security_groups_for_launch(self, security_group): + return security_group def create_volume(self, instance, vol_name, vol_size, vol_type='gp2'): return self.compute_session.create_volume( From 4a16f58776936127c09262674198f823cc92f5bd Mon Sep 17 00:00:00 2001 From: Amit Date: Fri, 22 Jan 2016 15:52:26 +0000 Subject: [PATCH 12/12] subnet lookup error --- gonzo/clouds/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gonzo/clouds/compute.py b/gonzo/clouds/compute.py index e78eb55..b6b3a5d 100644 --- a/gonzo/clouds/compute.py +++ b/gonzo/clouds/compute.py @@ -305,7 +305,7 @@ def get_subnet(self, subnet_id): for subnet in self.compute_session.ex_list_subnets(): if subnet.id == subnet_id: return subnet - raise LookupError("Subnet containting subnet: {} not found".format( + raise LookupError("Subnet with ID {} not found".format( subnet_id)) def get_security_group(self, group_name, subnet_id=None):