Bug 1361930 - support for NAT gateways, swing traffic for staging upl…

…oads that way (#314)
mozilla-releng · Sep 15, 2017 · 8111ab8 · 8111ab8
1 parent d23538d
commit 8111ab8
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 13 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -7,6 +7,6 @@ install:
     - travis_retry pip install tox==1.8
 script:
     - tox -e py27
-    - for f in `find . -name '*.json'` ./configs/watch_pending.cfg; do python -m json.tool $f; done
+    - for f in `find * -name '*.json'` ./configs/watch_pending.cfg; do python -m json.tool $f; done
 after_success:
     - tox -e py27-coveralls
diff --git a/cloudtools/scripts/aws_manage_routingtables.py b/cloudtools/scripts/aws_manage_routingtables.py
@@ -37,6 +37,14 @@ def resolve_host(hostname):
     return ips
 
 
+def get_nat_gateway(region):
+    # development shifted from boto to boto3 before NAT gateways were added to AWS :/
+    # https://github.com/boto/boto/issues/3443
+    import boto3
+    client = boto3.client('ec2', region_name=region)
+    return client.describe_nat_gateways()['NatGateways'][0]
+
+
 def sync_tables(conn, my_tables, remote_tables, aws_ranges):
     # Check that remote tables don't have overlapping names
     seen_names = set()
@@ -79,6 +87,7 @@ def sync_tables(conn, my_tables, remote_tables, aws_ranges):
         my_routes = set()
         IGW = None
         VGW = None
+        NAT = None
 
         # Resolve hostnames
         to_delete = set()
@@ -117,6 +126,12 @@ def sync_tables(conn, my_tables, remote_tables, aws_ranges):
                 if VGW is None:
                     VGW = conn.get_all_vpn_gateways()[0]
                 gateway_id = VGW.id
+            elif dest == "NAT":
+                # Use our VPC's NAT gateway
+                if NAT is None:
+                    log.info('Looking up NAT gateway')
+                    NAT = get_nat_gateway(conn.region.name)
+                gateway_id = NAT['NatGatewayId']
             elif dest == 'local':
                 gateway_id = 'local'
             elif dest and dest.startswith("i-"):

diff --git a/configs/routingtables.yml b/configs/routingtables.yml
@@ -22,9 +22,9 @@ us-west-2:
             upload.ffxbld.productdelivery.prod.mozaws.net: IGW
             upload.trybld.productdelivery.prod.mozaws.net: IGW
             upload.tbirdbld.productdelivery.prod.mozaws.net: IGW
-            upload.ffxbld.productdelivery.stage.mozaws.net: IGW
-            upload.trybld.productdelivery.stage.mozaws.net: IGW
-            upload.tbirdbld.productdelivery.stage.mozaws.net: IGW
+            upload.ffxbld.productdelivery.stage.mozaws.net: NAT
+            upload.trybld.productdelivery.stage.mozaws.net: NAT
+            upload.tbirdbld.productdelivery.stage.mozaws.net: NAT
             # seabld only here to appease firewall-test complexity, no harm
             upload.seabld.productdelivery.prod.mozaws.net: IGW
             upload.seabld.productdelivery.stage.mozaws.net: IGW
@@ -63,8 +63,8 @@ us-west-2:
             upload.ffxbld.productdelivery.prod.mozaws.net: IGW
             upload.tbirdbld.productdelivery.prod.mozaws.net: IGW
             upload.seabld.productdelivery.prod.mozaws.net: IGW
-            upload.ffxbld.productdelivery.stage.mozaws.net: IGW
-            upload.tbirdbld.productdelivery.stage.mozaws.net: IGW
+            upload.ffxbld.productdelivery.stage.mozaws.net: NAT
+            upload.tbirdbld.productdelivery.stage.mozaws.net: NAT
             upload.seabld.productdelivery.stage.mozaws.net: IGW
             crash-stats.mozilla.com: IGW
             queue.taskcluster.net: IGW
@@ -82,7 +82,7 @@ us-west-2:
             aus4.mozilla.org: IGW
             aus5.mozilla.org: IGW
             upload.trybld.productdelivery.prod.mozaws.net: IGW
-            upload.trybld.productdelivery.stage.mozaws.net: IGW
+            upload.trybld.productdelivery.stage.mozaws.net: NAT
             queue.taskcluster.net: IGW
 
             10.132.0.0/16: local
@@ -104,6 +104,12 @@ us-west-2:
             10.132.0.0/16: local
             0.0.0.0/0: VGW
 
+    upload-nat:
+        routes:
+            10.132.0.0/16: local
+            0.0.0.0/0: IGW
+
+
 us-east-1:
     default:
         routes:
@@ -120,9 +126,9 @@ us-east-1:
             upload.ffxbld.productdelivery.prod.mozaws.net: IGW
             upload.trybld.productdelivery.prod.mozaws.net: IGW
             upload.tbirdbld.productdelivery.prod.mozaws.net: IGW
-            upload.ffxbld.productdelivery.stage.mozaws.net: IGW
-            upload.trybld.productdelivery.stage.mozaws.net: IGW
-            upload.tbirdbld.productdelivery.stage.mozaws.net: IGW
+            upload.ffxbld.productdelivery.stage.mozaws.net: NAT
+            upload.trybld.productdelivery.stage.mozaws.net: NAT
+            upload.tbirdbld.productdelivery.stage.mozaws.net: NAT
             # seabld only here to appease firewall-test complexity, no harm
             upload.seabld.productdelivery.prod.mozaws.net: IGW
             upload.seabld.productdelivery.stage.mozaws.net: IGW
@@ -161,8 +167,8 @@ us-east-1:
             upload.ffxbld.productdelivery.prod.mozaws.net: IGW
             upload.tbirdbld.productdelivery.prod.mozaws.net: IGW
             upload.seabld.productdelivery.prod.mozaws.net: IGW
-            upload.ffxbld.productdelivery.stage.mozaws.net: IGW
-            upload.tbirdbld.productdelivery.stage.mozaws.net: IGW
+            upload.ffxbld.productdelivery.stage.mozaws.net: NAT
+            upload.tbirdbld.productdelivery.stage.mozaws.net: NAT
             upload.seabld.productdelivery.stage.mozaws.net: IGW
             crash-stats.mozilla.com: IGW
             queue.taskcluster.net: IGW
@@ -181,7 +187,7 @@ us-east-1:
             aus4.mozilla.org: IGW
             aus5.mozilla.org: IGW
             upload.trybld.productdelivery.prod.mozaws.net: IGW
-            upload.trybld.productdelivery.stage.mozaws.net: IGW
+            upload.trybld.productdelivery.stage.mozaws.net: NAT
             queue.taskcluster.net: IGW
 
             10.134.0.0/16: local
@@ -215,3 +221,8 @@ us-east-1:
 
             10.134.0.0/16: local
             0.0.0.0/0: VGW
+
+    upload-nat:
+        routes:
+            10.134.0.0/16: local
+            0.0.0.0/0: IGW
diff --git a/configs/subnets.yml b/configs/subnets.yml
@@ -48,6 +48,10 @@ us-west-2:
             name: signing
             routing_table: signing
 
+        10.132.31.0/24:
+            name: upload-nat
+            routing_table: upload-nat
+
 
 us-east-1:
     vpc-b42100df:
@@ -114,3 +118,7 @@ us-east-1:
         10.134.30.0/24:
             name: signing
             routing_table: signing
+
+        10.134.31.0/24:
+            name: upload-nat
+            routing_table: upload-nat
diff --git a/setup.py b/setup.py
@@ -33,6 +33,14 @@
         'cfn-pyplates>=0.5.0',
         'IPy==0.81',
         'redo==1.4',
+        'boto3==1.4.7',
+        'botocore==1.7.7',
+        'docutils==0.14',
+        'futures==3.1.1',
+        'jmespath==0.9.3',
+        'python-dateutil==2.6.0',
+        's3transfer==0.1.11',
+        'six==1.10.0',
     ],
     extras_require={
         'test': [