Cross AWS VPC Peering linking (#48)

cartography-cncf · Apr 12, 2019 · 68e8bb9 · 68e8bb9
1 parent 4054483
commit 68e8bb9
Show file tree

Hide file tree

Showing 7 changed files with 432 additions and 3 deletions.
diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher
@@ -40,4 +40,8 @@ CREATE INDEX ON :RDSInstance(id);
 CREATE INDEX ON :RDSInstance(db_instance_identifier);
 CREATE INDEX ON :S3Bucket(name);
 CREATE INDEX ON :User(arn);
-CREATE INDEX ON :S3Acl(id);
+CREATE INDEX ON :S3Acl(id);
+CREATE INDEX ON :AWSVpc(id);
+CREATE INDEX ON :AWSCidrBlock(id);
+CREATE INDEX ON :AWSIpv4CidrBlock(id);
+CREATE INDEX ON :AWSIpv6CidrBlock(id);
diff --git a/cartography/data/jobs/cleanup/aws_import_vpc_cleanup.json b/cartography/data/jobs/cleanup/aws_import_vpc_cleanup.json
@@ -0,0 +1,23 @@
+{
+  "statements": [{
+    "query": "MATCH (n:CidrBlock)<-[:BLOCK_ASSOCIATION]-(:AWSVpc)<-[:RESOURCE]-(:AWSAccount{id: {AWS_ID}}) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:CidrBlock)<-[r:BLOCK_ASSOCIATION]-(:AWSVpc)<-[:RESOURCE]-(:AWSAccount{id: {AWS_ID}}) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (n:AWSVpc)<-[:RESOURCE]-(:AWSAccount{id: {AWS_ID}}) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:AWSVpc)<-[r:RESOURCE]-(:AWSAccount{id: {AWS_ID}}) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  }],
+  "name": "cleanup AWS VPC information"
+}
diff --git a/cartography/data/jobs/cleanup/aws_import_vpc_peering_cleanup.json b/cartography/data/jobs/cleanup/aws_import_vpc_peering_cleanup.json
@@ -0,0 +1,33 @@
+{
+  "statements": [{
+    "query": "MATCH (n:AWSCidrBlock)<-[:BLOCK_ASSOCIATION]-(:AWSVpc)<-[:RESOURCE]-(:AWSAccount{foreign: true}) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:AWSCidrBlock)<-[r:BLOCK_ASSOCIATION]-(:AWSVpc)<-[:RESOURCE]-(:AWSAccount{foreign: true}) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (n:AWSVpc)<-[:RESOURCE]-(:AWSAccount{foreign: true}) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:AWSVpc)<-[r:RESOURCE]-(:AWSAccount{foreign: true}) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:AWSVpc)-[r:VPC_PEERING]-(:AWSVpc) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (n:AWSAccount{foreign: true}) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
+    "iterative": true,
+    "iterationsize": 100
+  }],
+  "name": "cleanup AWS VPC information"
+}
diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py
@@ -29,10 +29,12 @@ def _sync_one_account(session, boto3_session, account_id, regions, sync_tag, com
     # EC2
     # TODO move this to EC2 module
     logger.info("Syncing EC2 for account '%s'.", account_id)
+    ec2.sync_vpc(session, boto3_session, account_id, sync_tag, common_job_parameters)
     ec2.sync_ec2_security_groupinfo(session, boto3_session, regions, account_id, sync_tag, common_job_parameters)
     ec2.sync_ec2_instances(session, boto3_session, regions, account_id, sync_tag, common_job_parameters)
     ec2.sync_ec2_auto_scaling_groups(session, boto3_session, regions, account_id, sync_tag, common_job_parameters)
     ec2.sync_load_balancers(session, boto3_session, regions, account_id, sync_tag, common_job_parameters)
+    ec2.sync_vpc_peering(session, boto3_session, sync_tag, account_id, common_job_parameters)
 
     # RDS
     rds.sync_rds_instances(session, boto3_session, regions, account_id, sync_tag, common_job_parameters)