From 54f44ac9c9cb36fb5054c80e8f796fb5b2873fe1 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:49:36 +0000 Subject: [PATCH 01/30] added section for aws-centos6 --- metron-deployment/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 805aaf0f04..73f419595c 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -23,6 +23,8 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) + * [Single Node Cluster Deployment](#single-node-cluster-deployment) + * [10-node Cluster Deployment](#10-node-cluster-deployment) * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) @@ -112,6 +114,38 @@ To build the DEB packages, follow the instructions at [packaging/docker/deb-dock How do I deploy Metron within AWS? ---------------------------------- +You can deploy Metron as a single node or a 10-node cluster. The diffrence between both types of deployments are below. + +### Single Node Cluster Deployment +This will deploy Metron and all of its dependencies on a single EC2 node in AWS. + +#### What is this good for? + +* If you are new to Metron and want to explore the functionality that it offers, this is good place to start. + +* If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. + +* If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. + +* The cluster will survive a reboot. + +* The cluster can use pre-existing elastic ip id, security group id, and subnet id + +#### What is this **not** good for? + +* This VM is **not** intended for processing anything beyond the most basic, low volume work loads. + +* Additional services should **not** be installed along side Metron in this VM. + +* This VM should **not** be used to run a proof-of-concept for Apache Metron within your organization. + +Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. + +#### How? + +To deploy Metron in a VM running on your computer, follow the instructions at [development/aws-centos6](development/aws-centos6). + +### 10 Node Cluster Deployment This deploys Apache Metron on an automatically provisioned 10-node cluster running in Amazon Web Service's EC2 platform. From e95d73c8e94ff450aa9fa28af4db8ebce6c9c6ff Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:52:10 +0000 Subject: [PATCH 02/30] Update README.md --- metron-deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 73f419595c..e08fdea95e 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -23,8 +23,8 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) - * [Single Node Cluster Deployment](#single-node-cluster-deployment) - * [10-node Cluster Deployment](#10-node-cluster-deployment) + ...[AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + ...[AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) From bb91158a60edefd4263db598a109ee151d3a6cc1 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:52:21 +0000 Subject: [PATCH 03/30] Update README.md --- metron-deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index e08fdea95e..bed56bdd42 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -23,8 +23,8 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) - ...[AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) - ...[AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) + ... [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + ... [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) From f794c87c24e2dbdc24999bc25e0d35d7f553c6d8 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:53:15 +0000 Subject: [PATCH 04/30] Update README.md --- metron-deployment/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index bed56bdd42..031b7c0422 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -23,8 +23,11 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) + ... [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + ... [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) + * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) From dd149768690174d74d274832b1e99d4e56810072 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:53:26 +0000 Subject: [PATCH 05/30] Update README.md --- metron-deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 031b7c0422..f7f8794a29 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -24,9 +24,9 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I build DEB packages?](#how-do-i-build-deb-packages) * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) - ... [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + ...[AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) - ... [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) + ...[AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) From 977969966f4f7e7d8e71713b3819c9a378a99cb6 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:54:22 +0000 Subject: [PATCH 06/30] Update README.md --- metron-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index f7f8794a29..142bf40fd7 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -22,7 +22,7 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I deploy Metron on a single VM?](#how-do-i-deploy-metron-on-a-single-vm) * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) - * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) + * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws).. ...[AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) From ab403dd9c21e4a6b2fadddf393f2318e17cd7dd9 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:56:15 +0000 Subject: [PATCH 07/30] Update README.md --- metron-deployment/README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 142bf40fd7..8fd3f5405a 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -22,12 +22,9 @@ This project contains tools for building, packaging, and deploying Apache Metron * [How do I deploy Metron on a single VM?](#how-do-i-deploy-metron-on-a-single-vm) * [How do I build RPM packages?](#how-do-i-build-rpm-packages) * [How do I build DEB packages?](#how-do-i-build-deb-packages) - * [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws).. - - ...[AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) - - ...[AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) - + - [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) + - [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + - [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) From ce345e9b88b7e1376b328770701f5ced73a147a0 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 04:56:32 +0000 Subject: [PATCH 08/30] Update README.md --- metron-deployment/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 8fd3f5405a..260d17f35f 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -18,14 +18,14 @@ limitations under the License. This project contains tools for building, packaging, and deploying Apache Metron. Please refer to the following sections for more information on how to get Apache Metron running in your environment. - * [How do I deploy Metron with Ambari?](#how-do-i-deploy-metron-with-ambari) - * [How do I deploy Metron on a single VM?](#how-do-i-deploy-metron-on-a-single-vm) - * [How do I build RPM packages?](#how-do-i-build-rpm-packages) - * [How do I build DEB packages?](#how-do-i-build-deb-packages) + - [How do I deploy Metron with Ambari?](#how-do-i-deploy-metron-with-ambari) + - [How do I deploy Metron on a single VM?](#how-do-i-deploy-metron-on-a-single-vm) + - [How do I build RPM packages?](#how-do-i-build-rpm-packages) + - [How do I build DEB packages?](#how-do-i-build-deb-packages) - [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) - [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) - [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) - * [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) + - [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) How do I deploy Metron with Ambari? From a69c231d895ba40590316f71553acddada050293 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:00:53 +0000 Subject: [PATCH 09/30] Update README.md --- metron-deployment/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 260d17f35f..5a5f23c4d4 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -114,9 +114,9 @@ To build the DEB packages, follow the instructions at [packaging/docker/deb-dock How do I deploy Metron within AWS? ---------------------------------- -You can deploy Metron as a single node or a 10-node cluster. The diffrence between both types of deployments are below. +You can deploy Metron as a single node or as 10-node cluster in AWS EC2. The diffrence between deploying as a single node and as 10 nodes are below. -### Single Node Cluster Deployment +### AWS Single Node Cluster Deployment This will deploy Metron and all of its dependencies on a single EC2 node in AWS. #### What is this good for? @@ -125,11 +125,9 @@ This will deploy Metron and all of its dependencies on a single EC2 node in AWS. * If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. -* If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. - -* The cluster will survive a reboot. +* The single node will survive a reboot. -* The cluster can use pre-existing elastic ip id, security group id, and subnet id +* The single node can use pre-existing AWS elastic ip, security group id, and subnet id. #### What is this **not** good for? @@ -145,7 +143,7 @@ Running Metron within the resource constraints of a single VM is incredibly chal To deploy Metron in a VM running on your computer, follow the instructions at [development/aws-centos6](development/aws-centos6). -### 10 Node Cluster Deployment +### AWS 10 Node Cluster Deployment This deploys Apache Metron on an automatically provisioned 10-node cluster running in Amazon Web Service's EC2 platform. From 6cf22e6e61735dcd99b9940d319a2811ac758e63 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:04:21 +0000 Subject: [PATCH 10/30] Update README.md --- metron-deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 5a5f23c4d4..7ea68db803 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -117,7 +117,7 @@ How do I deploy Metron within AWS? You can deploy Metron as a single node or as 10-node cluster in AWS EC2. The diffrence between deploying as a single node and as 10 nodes are below. ### AWS Single Node Cluster Deployment -This will deploy Metron and all of its dependencies on a single EC2 node in AWS. +This will deploy Metron and all of its dependencies on a single node in Amazon Web Service's EC2 platform. #### What is this good for? @@ -127,7 +127,7 @@ This will deploy Metron and all of its dependencies on a single EC2 node in AWS. * The single node will survive a reboot. -* The single node can use pre-existing AWS elastic ip, security group id, and subnet id. +* The single node can use a pre-existing AWS elastic ip, security group id, and subnet id. #### What is this **not** good for? From 8a150df08440fa39678ba8826ce14afdd456552d Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:15:13 +0000 Subject: [PATCH 11/30] Created Vagrantfile to deploy to aws with centos6 --- .../development/aws-centos6/Vagrantfile | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 metron-deployment/development/aws-centos6/Vagrantfile diff --git a/metron-deployment/development/aws-centos6/Vagrantfile b/metron-deployment/development/aws-centos6/Vagrantfile new file mode 100644 index 0000000000..bfd73fb573 --- /dev/null +++ b/metron-deployment/development/aws-centos6/Vagrantfile @@ -0,0 +1,186 @@ +#Global Cybersecurity Resource 2017-2018 (cugcr.ca) +# +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +#a distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## +#1) ABOUT: +#------------------- +#This Vagrant file will deploy Apache Metron as a single node (node1) to AWS EC2. +# +#This file was originally adapted from the Metron Single Node Install: +########################################################################## +#2) LOCATION OF FILE(Where it should be placed): +#------------------- +#This file should be located in the following directory: +#/../metron-deployment/development/aws-centos6 +########################################################################## +#3) REQUIREMENTS (What is needed to run this file): +#------------------- +#install: vagrant plugin install vagrant-aws +#install: vagrant plugin install vagrant-reload +#install: vagrant plugin install vagrant-hostmanager +#install: pip install --upgrade setuptools --user python +# +#You will also need to reserve an AWS Elastic IP, a Subnet id, Security Group id, and a key pair (key pair name & *.pem file[remember to set permissions to chmod 400]). +# +########################################################################## +#4) BEFORE RUNNING: +#Recommended steps before running the vagrant file +#------------------- +#4.1) Clear Vagrant, Docker & Maven. Create new AWS Vagrant box: +#WARNING - THESE STEPS WILL DISTROY ALL LOCAL DOCKER CONTAINERS AND VAGRANT BOXES +#------------------- +#vagrant halt node1 -f +#vagrant halt default -f +#vagrant destroy node1 -f +#vagrant destroy default -f +#for i in `vagrant global-status | grep virtualbox | awk '{print $1 }'` ; do vagrant destroy $i ; done +#vagrant global-status --prune +#docker rm $(docker ps -aq) +#osascript -e 'quit app "Docker"' +#open -a Docker +#rm -rf /../.m2/repository/* +#rm -rf /../.vagrant.d/boxes/* +#vagrant box add dummy --force https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +# +#------------------- +#4.2) Set Environment Variables +#------------------- +#export AWS_ACCESS_KEY_ID='' +#export AWS_SECRET_ACCESS_KEY='' +# +#export AWS_KEYNAME='' +#export AWS_KEYPATH='../..*.pem' +# +#export AWS_ELASTIC_IP="" +#export AWS_SECURITYGROUP_ID="" +#export AWS_SUBNET_ID="" +# +#------------------- +#4.3) Update /etc/hosts file to include AWS_ELASTIC_IP which will be tied to "node1" (do only once) +#------------------- +#sed -i "$AWS_ELASTIC_IP node1" /etc/hosts +########################################################################## +#5) HOW TO EXECUTE/RUN THIS VAGRANT FILE: +#------------------- +#vagrant up --provider=aws +########################################################################## +#6) OTHER NOTES: +#------------------- +#How to find Canadian Centos7 AMI ids +#------------------- +#pip install awscli +#aws --region ca-central-1 ec2 describe-images --owners aws-marketplace --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce +# +#imageIDs: ami-28823e4c, ami-af62d0cb, ami-cbd069af + +require 'getoptlong' +require 'vagrant-aws' +require 'vagrant-reload' + +ansibleTags='' +ansibleSkipTags='sensors' + +begin + opts = GetoptLong.new( + [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ], + [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ] + ) + + opts.quiet = TRUE + + opts.each do |opt, arg| + case opt + when '--ansible-tags' + ansibleTags=arg + when '--ansible-skip-tags' + ansibleSkipTags=arg + end + end +rescue Exception => ignored +#Ignore to allow other opts to be passed to Vagrant +end + +puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != '' +puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != '' + +hosts = [{ + hostname: "node1", + ip: ENV['AWS_ELASTICIP'], + memory: "8192", + cpus: 4, + promisc: 2 # enables promisc on the 'Nth' network interface +}] + +Vagrant.configure(2) do |config| + config.vm.provider :aws do |aws, override| + aws.access_key_id = ENV['AWS_ACCESS_KEY_ID'] + aws.secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] + aws.keypair_name = ENV['AWS_KEYNAME'] + + #Below is a CentOS ami id available in Canada + aws.ami = "ami-00e45864" + aws.region = "ca-central-1" + aws.instance_type = "t2.2xlarge" + + #Example of a Security Group id: sg-1193xxxx + aws.security_groups = ENV['AWS_SECURITYGROUPID'] + + #Example of Subnet id: subnet-fbc3exxxx + aws.subnet_id = ENV['AWS_SUBNETID'] + aws.elastic_ip = ENV['AWS_ELASTICIP'] + override.vm.box = "dummy" + override.ssh.username = "centos" + override.ssh.private_key_path = ENV['AWS_KEYPATH'] + override.nfs.functional = false + aws.tags = { + 'Name' => 'Metron-VagrantAWS-SingleNode', + 'Organization' => 'GCR', + 'Developer' => 'GCR Dev Team', + 'Function' => 'MetronInstall' + } + aws.block_device_mapping = [{'DeviceName' =>'/dev/sda1', 'Ebs.VolumeSize' => 100, 'Ebs.DeleteOnTermination' => true}] + end + + #The following will install rsyslog, change hostname to node1, resize the disk partition, and reboot + config.vm.provision "shell", inline: <<-SHELL + yum install -y wget + wget http://rpms.adiscon.com/v8-stable/rsyslog.repo + mv rsyslog.repo /etc/yum.repos.d/rsyslog.repo + yum info rsyslog --skip-broken + yum install -y rsyslog + hostname node1 + sed -i "s/^HOSTNAME=.*/HOSTNAME=node1/g" /etc/sysconfig/network + sed -i "127.0.0.1 node1" /etc/hosts + echo "127.0.0.1 node1" >> /etc/hosts + echo -e "u s\nd 1\nn\np\n1\n2048\n\na\n1\nw\n" | fdisk /dev/xvda + ip link set eth0 promisc on + ifconfig eth0 promisc + shutdown -r now + SHELL + + config.vm.provision :reload + + config.vm.provision :ansible do |ansible| + ansible.playbook = "../../ansible/playbooks/metron_full_install.yml" + ansible.host_key_checking = false + ansible.limit = 'all' + ansible.sudo = true + ansible.tags = ansibleTags.split(",") if ansibleTags != '' + ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' + ansible.inventory_path = "ansible/inventory" + end +end From 4c96c08ca766280e421a94eb6d88769e4f0bf461 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:16:07 +0000 Subject: [PATCH 12/30] Create ansible.cfg --- .../development/aws-centos6/ansible.cfg | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 metron-deployment/development/aws-centos6/ansible.cfg diff --git a/metron-deployment/development/aws-centos6/ansible.cfg b/metron-deployment/development/aws-centos6/ansible.cfg new file mode 100644 index 0000000000..ade50ae8af --- /dev/null +++ b/metron-deployment/development/aws-centos6/ansible.cfg @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[defaults] +host_key_checking = false +library = ../../ansible/extra_modules +roles_path = ../../ansible/roles +pipelining = True +log_path = ./ansible.log + + +# fix for "ssh throws 'unix domain socket too long' " problem +[ssh_connection] +control_path = %(directory)s/%%h-%%p-%%r From 21ed0ee7a01775cd9021f70760e94ec7df824f1e Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:17:29 +0000 Subject: [PATCH 13/30] Create README.md --- .../development/aws-centos6/README.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 metron-deployment/development/aws-centos6/README.md diff --git a/metron-deployment/development/aws-centos6/README.md b/metron-deployment/development/aws-centos6/README.md new file mode 100644 index 0000000000..1bca39735a --- /dev/null +++ b/metron-deployment/development/aws-centos6/README.md @@ -0,0 +1,105 @@ + +Metron in AWS as a Single Node with CentOS 6 +================== + +This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running CentOS 6. + +Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. + +Getting Started +--------------- + +### Prerequisites + +The computer used to deploy Apache Metron will need to have the following components installed. + + - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0) + - [Docker](https://www.docker.com/community-edition) + - [Vagrant](https://www.vagrantup.com) 1.8+ + - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager) + - [Virtualbox](https://virtualbox.org) 5.0+ + - Python 2.7 + - Maven 3.3.9 + - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11) + +Running the following script can help validate whether you have all the prerequisites installed and running correctly. + + ``` + metron-deployment/scripts/platform-info.sh + ``` + +#### How do I install these on MacOS? + +Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project. + +1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/). + +1. Run the following command in a terminal to install all of the required tools. + + ``` + brew cask install vagrant virtualbox docker + brew cask install caskroom/versions/java8 + brew install maven@3.3 git + pip install ansible==2.2.2.0 + vagrant plugin install vagrant-hostmanager + open /Applications/Docker.app + ``` + +### Deploy Metron + +1. Ensure that the Docker service is running. + +1. Deploy Metron + + ``` + cd metron-deployment/development/centos6 + vagrant up + ``` + + Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host. + + ``` + vagrant provision + ``` + +### Explore Metron + +Navigate to the following resources to explore your newly minted Apache Metron environment. + +* [Metron Alerts](http://node1:4201) +* [Ambari](http://node1:8080) + +Connecting to the host through SSH is as simple as running the following command. +``` +vagrant ssh +``` + +### Working with Metron + +In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components. + +``` +vagrant --ansible-tags="sensor-stubs" provision +``` + +Tags are listed in the playbooks, some frequently used tags: ++ `hdp-install` - Install HDP ++ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) ++ `sensors` - Deploy and start the sensors. ++ `sensor-stubs` - Deploy and start the sensor stubs. From 04d5d1b7e574f7d1bad9183d4d9228f8dd367218 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:18:26 +0000 Subject: [PATCH 14/30] Create hosts --- .../development/aws-centos6/inventory/hosts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 metron-deployment/development/aws-centos6/inventory/hosts diff --git a/metron-deployment/development/aws-centos6/inventory/hosts b/metron-deployment/development/aws-centos6/inventory/hosts new file mode 100644 index 0000000000..9bd9ea1e05 --- /dev/null +++ b/metron-deployment/development/aws-centos6/inventory/hosts @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[ambari_master] +node1 + +[ambari_slave] +node1 + +[metron] +node1 + +[search] +node1 + +[sensors] +node1 + +[pcap_server] +node1 + +[web] +node1 + +[zeppelin] +node1 + +[monit:children] +sensors +pcap_server + +[local] +127.0.0.1 From 7b1b2b2b65f80666c4fd02c7f5acdb94c05bffbc Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 05:19:25 +0000 Subject: [PATCH 15/30] Create all --- .../aws-centos6/inventory/group_vars/all | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 metron-deployment/development/aws-centos6/inventory/group_vars/all diff --git a/metron-deployment/development/aws-centos6/inventory/group_vars/all b/metron-deployment/development/aws-centos6/inventory/group_vars/all new file mode 100644 index 0000000000..7e162aef77 --- /dev/null +++ b/metron-deployment/development/aws-centos6/inventory/group_vars/all @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# only need to build the RPMs for CentOS +metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms" + +# ambari +ambari_host: "{{ groups.ambari_master[0] }}" +hdp_host_group: "{{ groups.ambari_slave }}" +ambari_port: 8080 +ambari_user: admin +ambari_password: admin +cluster_type: single_node_vm +ambari_server_mem: 512 +java_home: /usr/jdk64/jdk1.8.0_77 + +# hbase +pcap_hbase_table: pcap +tracker_hbase_table: access_tracker +threatintel_hbase_table: threatintel +enrichment_hbase_table: enrichment + +# metron +metron_version: 0.4.3 +metron_directory: /usr/metron/{{ metron_version }} +bro_version: "2.5.2" +fixbuf_version: "1.7.1" +yaf_version: "2.8.0" +daq_version: "2.0.6-1" +pycapa_repo: "https://github.com/OpenSOC/pycapa.git" +pycapa_home: "/opt/pycapa" +snort_version: "2.9.8.0-1" +snort_alert_csv_path: "/var/log/snort/alert.csv" +threat_intel_bulk_load: False + +# data directories - only required to override defaults +zookeeper_data_dir: "/data1/hadoop/zookeeper" +namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary" +namenode_name_dir: "/data1/hadoop/hdfs/namenode" +datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data" +journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode" +nodemanager_local_dirs: "/data1/hadoop/yarn/local" +timeline_ldb_store_path: "/data1/hadoop/yarn/timeline" +timeline_ldb_state_path: "/data1/hadoop/yarn/timeline" +nodemanager_log_dirs: "/data1/hadoop/yarn/log" +jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs" +storm_local_dir: "/data1/hadoop/storm" +kafka_log_dirs: "/data1/kafka-log" +elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" + +# sensors +sensor_test_mode: True +install_pycapa: False +install_bro: True +install_snort: True +install_yaf: False +install_pcap_replay: True +sniff_interface: eth0 +pcap_replay_interface: "{{ sniff_interface }}" +pcapservice_port: 8081 + +# search +install_elasticsearch: True +install_solr: False +solr_collection_name: Metron +solr_number_shards: 1 +solr_replication_factor: 1 +elasticsearch_transport_port: 9300 +## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores. +elasticsearch_network_interface: _eth0_ +elasticsearch_web_port: 9200 From 45025382e5dfce67b6b88352ac363c9cd298e390 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 06:43:12 +0000 Subject: [PATCH 16/30] added AWS AMI instructions (3rd option for deployment) --- metron-deployment/README.md | 97 ++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 7ea68db803..117f59a426 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -23,7 +23,8 @@ This project contains tools for building, packaging, and deploying Apache Metron - [How do I build RPM packages?](#how-do-i-build-rpm-packages) - [How do I build DEB packages?](#how-do-i-build-deb-packages) - [How do I deploy Metron within AWS?](#how-do-i-deploy-metron-within-aws) - - [AWS Single Node Cluster Deployment](#aws-single-node-cluster-deployment) + - [AWS Single Node Cluster Deployment Using Vagrant](#aws-single-node-cluster-deployment-using-vagrant) + - [AWS Single Node Cluster Deployment Using an AMI](#aws-single-node-cluster-deployment-using-an-ami) - [AWS 10 Node Cluster Deployment](#aws-10-node-cluster-deployment) - [How do I build Metron with Docker?](#how-do-i-build-metron-with-docker) @@ -114,7 +115,97 @@ To build the DEB packages, follow the instructions at [packaging/docker/deb-dock How do I deploy Metron within AWS? ---------------------------------- -You can deploy Metron as a single node or as 10-node cluster in AWS EC2. The diffrence between deploying as a single node and as 10 nodes are below. +You can deploy Metron into Amazon Web Service in three ways: +i) As a single node using Vagrant +ii) As single node using an AMI from the AWS Marketplace +iii) As a 10-node cluster in AWS EC2 + +Below will provide more information on the three different deployment methods + +### AWS Single Node Cluster Deployment Using Vagrant +This will deploy Metron and all of its dependencies as a single node in Amazon Web Service's EC2 platform using Vagrant. + +#### What is this good for? + +* If you are new to Metron and want to explore the functionality that it offers, this is good place to start. + +* If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. + +* The single node will survive a reboot. + +* The single node can use a pre-existing AWS elastic ip, security group id, and subnet id. + +#### What is this **not** good for? + +* This single node is **not** intended for processing anything beyond the most basic, low volume work loads. + +* This single node is **not** intended for processing anything beyond the most basic, low volume work loads. + +* Additional services should **not** be installed along side Metron in this VM. + +* This single node should **not** be used to run a proof-of-concept for Apache Metron within your organization. + +* You might need to run the Vagrant file in Mac OS and have install prerequisites installed properly + +Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. + +#### How? + +To deploy Metron in a EC2 as a single node using Vagrant, follow the instructions at [development/aws-centos6](development/aws-centos6). + + +### AWS Single Node Cluster Deployment Using an AMI +This will deploy Metron as a single node in Amazon Web Service's EC2 platform by using existing Amazon Machine Image (AMI) that can be found in the AWS Marketplace. + +#### What is this good for? + +* This is intended to be the simplest EC2 AWS deployment option + +* No need to have a separate machine to deploy + +* No need for preinstall requriments + +* If you are new to Metron and want to explore the functionality that it offers, this is good place to start. + +* The single node will survive a reboot. + +* The single node can use your pre-existing AWS infrastructure settings (example: security group, keys ext..) + +* If you are a developer contributing to the Apache Metron project, you will see your changes if making modifications after deployment + +#### What is this **not** good for? + +* This single node is **not** intended for processing anything beyond the most basic, low volume work loads. + +* Additional services should **not** be installed along side Metron in this VM. + +* This single node should **not** be used to run a proof-of-concept for Apache Metron within your organization. + +* You might need to run the Vagrant file in Mac OS and have install prerequisites installed properly + +Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. + +#### How? + +1) In the EC2 Dashboard click on "Launch an instance" in the "Canada (Central)" region +2) Search for "GCR-Xetron Demo" in the "AWS Marketplace" and click on "Select" +3) Manually choose the following non-default options +t2.t2xlarge +4) Launch the instance +5) Associate the newly launched instance to an elastic IP(optional) +6) After the image is launched you will need to change the /etc/hosts file. + +SSH into the machine using your \*.pem key +``` +ssh -i ".pem" centos@ +``` + +Update the /etc/hosts file to look like the following +``` +127.0.0.1 localhost node +``` +7) Restart the instance + ### AWS Single Node Cluster Deployment This will deploy Metron and all of its dependencies on a single node in Amazon Web Service's EC2 platform. @@ -143,6 +234,8 @@ Running Metron within the resource constraints of a single VM is incredibly chal To deploy Metron in a VM running on your computer, follow the instructions at [development/aws-centos6](development/aws-centos6). + + ### AWS 10 Node Cluster Deployment This deploys Apache Metron on an automatically provisioned 10-node cluster running in Amazon Web Service's EC2 platform. From 0a22ae2bc7e21345a062a76e9cd187271ee5a041 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 06:54:23 +0000 Subject: [PATCH 17/30] Update README.md --- metron-deployment/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 117f59a426..49feb6a7ee 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -115,10 +115,10 @@ To build the DEB packages, follow the instructions at [packaging/docker/deb-dock How do I deploy Metron within AWS? ---------------------------------- -You can deploy Metron into Amazon Web Service in three ways: -i) As a single node using Vagrant -ii) As single node using an AMI from the AWS Marketplace -iii) As a 10-node cluster in AWS EC2 +You can deploy Metron into Amazon Web Service(AWS) in three ways: +i) [As a single node using Vagrant](#aws-single-node-cluster-deployment-using-vagrant) +ii) [As single node using an AMI from the AWS Marketplace](#aws-single-node-cluster-deployment-using-an-ami) +iii) [As a 10-node cluster](#aws-10-node-cluster-deployment) Below will provide more information on the three different deployment methods @@ -151,7 +151,7 @@ Running Metron within the resource constraints of a single VM is incredibly chal #### How? -To deploy Metron in a EC2 as a single node using Vagrant, follow the instructions at [development/aws-centos6](development/aws-centos6). +To deploy Metron in EC2 as a single node using Vagrant, follow the instructions at [development/aws-centos6](development/aws-centos6). ### AWS Single Node Cluster Deployment Using an AMI @@ -187,11 +187,12 @@ Running Metron within the resource constraints of a single VM is incredibly chal #### How? -1) In the EC2 Dashboard click on "Launch an instance" in the "Canada (Central)" region +1) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region 2) Search for "GCR-Xetron Demo" in the "AWS Marketplace" and click on "Select" -3) Manually choose the following non-default options +3) Manually choose the following mandatory non-default options t2.t2xlarge 4) Launch the instance +6) Change security group setting to only allow traffic to what is necessary 5) Associate the newly launched instance to an elastic IP(optional) 6) After the image is launched you will need to change the /etc/hosts file. @@ -205,7 +206,8 @@ Update the /etc/hosts file to look like the following 127.0.0.1 localhost node ``` 7) Restart the instance - +8) Go to the following to see the Metron dashboard +http://:5000 ### AWS Single Node Cluster Deployment This will deploy Metron and all of its dependencies on a single node in Amazon Web Service's EC2 platform. From 77a674989c3a8ea16f47fe56d9ad0ebbd283fe0e Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 06:58:37 +0000 Subject: [PATCH 18/30] added AMI id ami-93cb4ff7 --- metron-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 49feb6a7ee..f7f1b9a6b1 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -188,7 +188,7 @@ Running Metron within the resource constraints of a single VM is incredibly chal #### How? 1) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region -2) Search for "GCR-Xetron Demo" in the "AWS Marketplace" and click on "Select" +2) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "AWS Marketplace" and click on "Select" 3) Manually choose the following mandatory non-default options t2.t2xlarge 4) Launch the instance From fdec5840d37d8549f42c52a53b400c32ed9d5f24 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 07:53:44 +0000 Subject: [PATCH 19/30] Update README.md --- metron-deployment/README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index f7f1b9a6b1..a572fc3593 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -117,7 +117,7 @@ How do I deploy Metron within AWS? ---------------------------------- You can deploy Metron into Amazon Web Service(AWS) in three ways: i) [As a single node using Vagrant](#aws-single-node-cluster-deployment-using-vagrant) -ii) [As single node using an AMI from the AWS Marketplace](#aws-single-node-cluster-deployment-using-an-ami) +ii) [As single node using an AMI from the AWS Community AMI Marketplace](#aws-single-node-cluster-deployment-using-an-ami) iii) [As a 10-node cluster](#aws-10-node-cluster-deployment) Below will provide more information on the three different deployment methods @@ -155,7 +155,7 @@ To deploy Metron in EC2 as a single node using Vagrant, follow the instructions ### AWS Single Node Cluster Deployment Using an AMI -This will deploy Metron as a single node in Amazon Web Service's EC2 platform by using existing Amazon Machine Image (AMI) that can be found in the AWS Marketplace. +This will deploy Metron as a single node in Amazon Web Service's EC2 platform by using existing Amazon Machine Image (AMI) that can be found in the AWS Community AMI Marketplace. #### What is this good for? @@ -188,26 +188,24 @@ Running Metron within the resource constraints of a single VM is incredibly chal #### How? 1) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region -2) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "AWS Marketplace" and click on "Select" -3) Manually choose the following mandatory non-default options +2) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "Community AMIs" and click on "Select" +3) Manually choose the following mandatory/non-default option and then "Launch Instance" t2.t2xlarge -4) Launch the instance -6) Change security group setting to only allow traffic to what is necessary -5) Associate the newly launched instance to an elastic IP(optional) -6) After the image is launched you will need to change the /etc/hosts file. +5) Change security group setting to only allow traffic to what is necessary. By default a new security group might block all inbound traffic except SSH. Inbound and outbound ports 8080, 5000, 4200 ext.. will need to be allowed for your web client. +6) (optional)Associate the newly launched instance to an elastic IP +7) After the instance is launched you will need to change the hostname to node1. SSH into the machine using your \*.pem key ``` ssh -i ".pem" centos@ ``` -Update the /etc/hosts file to look like the following +Update the /etc/sysconfig/network file to look like the following ``` -127.0.0.1 localhost node +sudo sed -i "s/^HOSTNAME=.*/HOSTNAME=node1/g" /etc/sysconfig/network ``` -7) Restart the instance -8) Go to the following to see the Metron dashboard -http://:5000 +8) Reboot the instance +9) Go to the Ambari dashboard (http://:8080) and select "Start All" services in the left dropdown menu ### AWS Single Node Cluster Deployment This will deploy Metron and all of its dependencies on a single node in Amazon Web Service's EC2 platform. From 0c435cf7b76a0a290eb5b1e85a1a50e50eee2e09 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:05:46 +0000 Subject: [PATCH 20/30] Update README.md --- .../development/aws-centos6/README.md | 79 ++++++++++--------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/metron-deployment/development/aws-centos6/README.md b/metron-deployment/development/aws-centos6/README.md index 1bca39735a..c21a88b43b 100644 --- a/metron-deployment/development/aws-centos6/README.md +++ b/metron-deployment/development/aws-centos6/README.md @@ -15,12 +15,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -Metron in AWS as a Single Node with CentOS 6 +Metron in AWS as a Single Node with CentOS 6 using Vagrant ================== -This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running CentOS 6. - -Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. +This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized node in AWS EC2. Getting Started --------------- @@ -29,7 +27,7 @@ Getting Started The computer used to deploy Apache Metron will need to have the following components installed. - - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0) + - [Ansible](https://github.com/ansible/ansible) (2.2.2.0) - [Docker](https://www.docker.com/community-edition) - [Vagrant](https://www.vagrantup.com) 1.8+ - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager) @@ -50,7 +48,7 @@ Any platform that supports these tools is suitable, but the following instructio 1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/). -1. Run the following command in a terminal to install all of the required tools. +2. Run the following commands in a terminal to install all of the required tools. ``` brew cask install vagrant virtualbox docker @@ -58,48 +56,57 @@ Any platform that supports these tools is suitable, but the following instructio brew install maven@3.3 git pip install ansible==2.2.2.0 vagrant plugin install vagrant-hostmanager + vagrant plugin install vagrant-aws + vagrant plugin install vagrant-reload + pip install --upgrade setuptools --user python open /Applications/Docker.app ``` +3. In your AWS console you need to reserve/create an AWS Elastic IP, a Subnet id, Security Group id, and a key pair (key pair name & *.pem file[remember to set permissions to chmod 400]). + +4. The following will clear existing Vagrant, Docker & Maven deployments. WARNING - THESE STEPS WILL DISTROY ALL LOCAL DOCKER CONTAINERS AND VAGRANT BOXES +``` +vagrant halt node1 -f +vagrant halt default -f +vagrant destroy node1 -f +vagrant destroy default -f +for i in `vagrant global-status | grep virtualbox | awk '{print $1 }'` ; do vagrant destroy $i ; done +vagrant global-status --prune +docker rm $(docker ps -aq) +osascript -e 'quit app "Docker"' +open -a Docker +rm -rf /../.m2/repository/* +rm -rf /../.vagrant.d/boxes/* +vagrant box add dummy --force https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +``` +5. Associate your AWS ids to the following enviroment variables names +``` +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export AWS_KEYNAME='' +export AWS_KEYPATH='../..*.pem' +export AWS_ELASTIC_IP="" +export AWS_SECURITYGROUP_ID="" +export AWS_SUBNET_ID="" +``` +6. Update your local /etc/hosts file to include AWS_ELASTIC_IP which will be tied to "node1" (do only once). +``` +sed -i "$AWS_ELASTIC_IP node1" /etc/hosts +``` ### Deploy Metron 1. Ensure that the Docker service is running. -1. Deploy Metron +2. Deploy Metron ``` - cd metron-deployment/development/centos6 - vagrant up - ``` - - Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host. - - ``` - vagrant provision + cd metron-deployment/development/aws-centos6 + vagrant up --provider=aws ``` ### Explore Metron Navigate to the following resources to explore your newly minted Apache Metron environment. -* [Metron Alerts](http://node1:4201) -* [Ambari](http://node1:8080) - -Connecting to the host through SSH is as simple as running the following command. -``` -vagrant ssh -``` - -### Working with Metron - -In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components. - -``` -vagrant --ansible-tags="sensor-stubs" provision -``` - -Tags are listed in the playbooks, some frequently used tags: -+ `hdp-install` - Install HDP -+ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. -+ `sensor-stubs` - Deploy and start the sensor stubs. +* [Metron Alerts](http://:4201) +* [Ambari](http://:8080) From 5b77997591731de17235d2a8014a437c64ac81aa Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:08:09 +0000 Subject: [PATCH 21/30] Update README.md --- metron-deployment/development/aws-centos6/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-deployment/development/aws-centos6/README.md b/metron-deployment/development/aws-centos6/README.md index c21a88b43b..ad19ce0b8a 100644 --- a/metron-deployment/development/aws-centos6/README.md +++ b/metron-deployment/development/aws-centos6/README.md @@ -63,7 +63,7 @@ Any platform that supports these tools is suitable, but the following instructio ``` 3. In your AWS console you need to reserve/create an AWS Elastic IP, a Subnet id, Security Group id, and a key pair (key pair name & *.pem file[remember to set permissions to chmod 400]). -4. The following will clear existing Vagrant, Docker & Maven deployments. WARNING - THESE STEPS WILL DISTROY ALL LOCAL DOCKER CONTAINERS AND VAGRANT BOXES +4. The following will clear existing Vagrant, Docker & Maven builds and deployments. WARNING - THESE STEPS WILL DISTROY ALL LOCAL DOCKER CONTAINERS AND VAGRANT BOXES. It will also create a new Vagrant box. ``` vagrant halt node1 -f vagrant halt default -f @@ -78,7 +78,7 @@ rm -rf /../.m2/repository/* rm -rf /../.vagrant.d/boxes/* vagrant box add dummy --force https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box ``` -5. Associate your AWS ids to the following enviroment variables names +5. Associate your AWS ids and key information to the following enviroment variables (enter values inside the quotes). ``` export AWS_ACCESS_KEY_ID='' export AWS_SECRET_ACCESS_KEY='' From 6830b75f1b657e72bcc2b8ab055a4771e9fda869 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:12:13 +0000 Subject: [PATCH 22/30] Added $ warning, Accepting Centos6 AMI Terms, Intro to AWS --- .../development/aws-centos6/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/metron-deployment/development/aws-centos6/README.md b/metron-deployment/development/aws-centos6/README.md index ad19ce0b8a..1cd96374c1 100644 --- a/metron-deployment/development/aws-centos6/README.md +++ b/metron-deployment/development/aws-centos6/README.md @@ -20,8 +20,26 @@ Metron in AWS as a Single Node with CentOS 6 using Vagrant This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized node in AWS EC2. +Warning: Amazon will charge for the use of their resources when running Apache Metron. The amount will vary based on the number and size of hosts, along with current Amazon pricing structure. Be sure to stop or terminate all of the hosts instantiated by Apache Metron when not in use to avoid unnecessary charges. + Getting Started --------------- +### Amazon Web Services + +If you already have an Amazon Web Services account that you have used to deploy EC2 hosts, then you should be able to skip the next few steps. + +1. Head over to [Amazon Web Services](http://aws.amazon.com/) and create an account. As part of the account creation process you will need to provide a credit card to cover any charges that may apply. + +2. Create a set of user credentials through [Amazon's Identity and Access Management (IAM) ](https://console.aws.amazon.com/iam/) dashboard. On the IAM dashboard menu click "Users" and then "Create New User". Provide a name and ensure that "Generate an access key for each user" remains checked. Download the credentials and keep them for later use. + +3. While still in [Amazon's Identity and Access Management (IAM) ](https://console.aws.amazon.com/iam/) dashboard, click on the user that was previously created. Click the "Permissions" tab and then the "Attach Policy" button. Attach the following policies to the user. + + - AmazonEC2FullAccess + - AmazonVPCFullAccess + +4. Apache Metron uses the [official, open source CentOS 6](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) Amazon Machine Image (AMI). If you have never used this AMI before then you will need to accept Amazon's terms and conditions. Navigate to the [web page for this AMI](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) and click the "Continue" button. Choose the "Manual Launch" tab then click the "Accept Software Terms" button. + +Having successfully created your Amazon Web Services account, hopefully you will find that the most difficult tasks are behind us. ### Prerequisites From 7280d6807eb0dfe8703ea14c46099c7a9cdbc137 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:21:42 +0000 Subject: [PATCH 23/30] Added accepting terms for Centos6 AMI terms --- metron-deployment/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index a572fc3593..fa5575a892 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -120,7 +120,9 @@ i) [As a single node using Vagrant](#aws-single-node-cluster-deployment-using-va ii) [As single node using an AMI from the AWS Community AMI Marketplace](#aws-single-node-cluster-deployment-using-an-ami) iii) [As a 10-node cluster](#aws-10-node-cluster-deployment) -Below will provide more information on the three different deployment methods +Below will provide more information on the three different deployment methods. +Warning: Amazon will charge for the use of their resources when running Apache Metron. The amount will vary based on the number and size of hosts, along with current Amazon pricing structure. Be sure to stop or terminate all of the hosts instantiated by Apache Metron when not in use to avoid unnecessary charges. + ### AWS Single Node Cluster Deployment Using Vagrant This will deploy Metron and all of its dependencies as a single node in Amazon Web Service's EC2 platform using Vagrant. @@ -161,9 +163,7 @@ This will deploy Metron as a single node in Amazon Web Service's EC2 platform by * This is intended to be the simplest EC2 AWS deployment option -* No need to have a separate machine to deploy - -* No need for preinstall requriments +* No need for preinstall requriments on a seperate machine * If you are new to Metron and want to explore the functionality that it offers, this is good place to start. @@ -186,11 +186,11 @@ This will deploy Metron as a single node in Amazon Web Service's EC2 platform by Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. #### How? - -1) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region -2) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "Community AMIs" and click on "Select" -3) Manually choose the following mandatory/non-default option and then "Launch Instance" -t2.t2xlarge +1) Navigate to the [web page for this AMI](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) and "Accept Software Terms" for the "Manual Launch" tab. +2) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region +3) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "Community AMIs" and click on "Select" +4) Manually choose the following mandatory/non-default option and then "Launch Instance" +t2.t2xlarge [32 GB Memory, 8 Cores, 64 bit] 5) Change security group setting to only allow traffic to what is necessary. By default a new security group might block all inbound traffic except SSH. Inbound and outbound ports 8080, 5000, 4200 ext.. will need to be allowed for your web client. 6) (optional)Associate the newly launched instance to an elastic IP 7) After the instance is launched you will need to change the hostname to node1. From ce50d4132effe8b42a544339d208d6c1f32ba661 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:24:14 +0000 Subject: [PATCH 24/30] Update README.md --- metron-deployment/development/aws-centos6/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metron-deployment/development/aws-centos6/README.md b/metron-deployment/development/aws-centos6/README.md index 1cd96374c1..f4d4b1c4fb 100644 --- a/metron-deployment/development/aws-centos6/README.md +++ b/metron-deployment/development/aws-centos6/README.md @@ -37,11 +37,10 @@ If you already have an Amazon Web Services account that you have used to deploy - AmazonEC2FullAccess - AmazonVPCFullAccess -4. Apache Metron uses the [official, open source CentOS 6](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) Amazon Machine Image (AMI). If you have never used this AMI before then you will need to accept Amazon's terms and conditions. Navigate to the [web page for this AMI](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) and click the "Continue" button. Choose the "Manual Launch" tab then click the "Accept Software Terms" button. -Having successfully created your Amazon Web Services account, hopefully you will find that the most difficult tasks are behind us. ### Prerequisites +Apache Metron uses the [official, open source CentOS 6](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) Amazon Machine Image (AMI). If you have never used this AMI before then you will need to accept Amazon's terms and conditions. Navigate to the [web page for this AMI](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) and "Accept Software Terms" for the "Manual Launch" tab. The computer used to deploy Apache Metron will need to have the following components installed. From 5fd24e77e0504090dd74f9ea139a1e33f367a13c Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 08:31:24 +0000 Subject: [PATCH 25/30] added AWS using Vagrant and AWS using a AMI --- metron-deployment/development/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md index bc998094f1..30b72dd009 100644 --- a/metron-deployment/development/README.md +++ b/metron-deployment/development/README.md @@ -21,4 +21,6 @@ This directory contains environments useful for Metron developers. These enviro * Metron running on CentOS 6 * Metron running on Ubuntu 14 +* Metron running in AWS EC2 with CentOS 6 using Vagrant +* Metron running in AWS EC2 with CentOS 6 using an AMI * Fastcapa From eb4dac16a51136440276fde02dcfc100ffdb5915 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 09:06:27 +0000 Subject: [PATCH 26/30] Update README.md --- metron-deployment/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index fa5575a892..f373c9aece 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -190,6 +190,7 @@ Running Metron within the resource constraints of a single VM is incredibly chal 2) In the "EC2 Dashboard" click on "Launch Instance" in the "Canada (Central)" region 3) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "Community AMIs" and click on "Select" 4) Manually choose the following mandatory/non-default option and then "Launch Instance" + t2.t2xlarge [32 GB Memory, 8 Cores, 64 bit] 5) Change security group setting to only allow traffic to what is necessary. By default a new security group might block all inbound traffic except SSH. Inbound and outbound ports 8080, 5000, 4200 ext.. will need to be allowed for your web client. 6) (optional)Associate the newly launched instance to an elastic IP From f1e3ba6e8d6c36dc6b76d75af05f110b9db01091 Mon Sep 17 00:00:00 2001 From: as22323 Date: Tue, 30 Jan 2018 09:07:07 +0000 Subject: [PATCH 27/30] Update README.md --- metron-deployment/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metron-deployment/README.md b/metron-deployment/README.md index f373c9aece..79fd315f2b 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -191,7 +191,8 @@ Running Metron within the resource constraints of a single VM is incredibly chal 3) Search for "GCR-Xetron Demo" or "ami-93cb4ff7" in the "Community AMIs" and click on "Select" 4) Manually choose the following mandatory/non-default option and then "Launch Instance" -t2.t2xlarge [32 GB Memory, 8 Cores, 64 bit] +- t2.t2xlarge [32 GB Memory, 8 Cores, 64 bit] + 5) Change security group setting to only allow traffic to what is necessary. By default a new security group might block all inbound traffic except SSH. Inbound and outbound ports 8080, 5000, 4200 ext.. will need to be allowed for your web client. 6) (optional)Associate the newly launched instance to an elastic IP 7) After the instance is launched you will need to change the hostname to node1. From e939bfd24803285f9696af1a6752ccb763034c0c Mon Sep 17 00:00:00 2001 From: as22323 Date: Wed, 31 Jan 2018 22:20:03 +0000 Subject: [PATCH 28/30] Attemptting to change uppercase to lowercase (Metron UI) --- .../sensor-parser-config/sensor-parser-config.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html index a784436a0f..31a3335a3a 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html @@ -53,7 +53,7 @@
- +
From fb960fc8ad9d9ced81acd9e21eebe7edd81b76e8 Mon Sep 17 00:00:00 2001 From: as22323 Date: Wed, 4 Apr 2018 19:14:29 +0000 Subject: [PATCH 29/30] Create README.md --- .../0.4.1SingleNodeEC2Vagrant/README.md | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/README.md diff --git a/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/README.md b/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/README.md new file mode 100644 index 0000000000..77251222ee --- /dev/null +++ b/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/README.md @@ -0,0 +1,161 @@ + +Metron 0.4.1 - AWS EC2 Single Node Deployment with Vagrant +================== + +This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized node in AWS EC2. + +Warning: Amazon will charge for the use of their resources when running Apache Metron. + + +### AWS Single Node Cluster Deployment Using Vagrant +This will deploy Metron and all of its dependencies as a single node in Amazon Web Service's EC2 platform using Vagrant. + +#### What is this good for? + +* If you are new to Metron and want to explore the functionality that it offers, this is good place to start. + +* If you are a developer contributing to the Apache Metron project, this is also a great way to test your changes. + +* The single node will survive a reboot. + +* The single node can use a pre-existing AWS elastic ip, security group id, and subnet id. + +#### What is this **not** good for? + +* This single node is **not** intended for processing anything beyond the most basic, low volume work loads. + +* Additional services should **not** be installed along side Metron in this VM. + +* This single node should **not** be used to run a proof-of-concept for Apache Metron within your organization. + +* You might need to run the Vagrant file in Mac OS and have install prerequisites installed properly + +Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. + + + + + +Getting Started +--------------- +### Amazon Web Services + +If you already have an Amazon Web Services account that you have used to deploy EC2 hosts, then you should be able to skip the next few steps. + +1. Head over to [Amazon Web Services](http://aws.amazon.com/) and create an account. As part of the account creation process you will need to provide a credit card to cover any charges that may apply. + +2. Create a set of user credentials through [Amazon's Identity and Access Management (IAM) ](https://console.aws.amazon.com/iam/) dashboard. On the IAM dashboard menu click "Users" and then "Create New User". Provide a name and ensure that "Generate an access key for each user" remains checked. Download the credentials and keep them for later use. + +3. While still in [Amazon's Identity and Access Management (IAM) ](https://console.aws.amazon.com/iam/) dashboard, click on the user that was previously created. Click the "Permissions" tab and then the "Attach Policy" button. Attach the following policies to the user. + + - AmazonEC2FullAccess + - AmazonVPCFullAccess + + + +### Prerequisites +Apache Metron uses the [official, open source CentOS 6](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) Amazon Machine Image (AMI). If you have never used this AMI before then you will need to accept Amazon's terms and conditions. Navigate to the [web page for this AMI](https://aws.amazon.com/marketplace/pp/B00NQAYLWO) and "Accept Software Terms" for the "Manual Launch" tab. + +The computer used to deploy Apache Metron will need to have the following components installed. + + - [Ansible](https://github.com/ansible/ansible) (2.2.2.0) + - [Docker](https://www.docker.com/community-edition) + - [Vagrant](https://www.vagrantup.com) 1.8+ + - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager) + - [Virtualbox](https://virtualbox.org) 5.0+ + - Python 2.7 + - Maven 3.3.9 + - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11) + +Download the 0.4.1 build from [here]: + +Running the following script can help validate whether you have all the prerequisites installed and running correctly. + + ``` + metron-deployment/scripts/platform-info.sh + ``` + +#### How do I install these on MacOS? + +Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project. + +1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/). + +2. Run the following commands in a terminal to install all of the required tools. + + ``` + brew cask install vagrant virtualbox docker + brew cask install caskroom/versions/java8 + brew install maven@3.3 git + pip install ansible==2.2.2.0 + vagrant plugin install vagrant-hostmanager + vagrant plugin install vagrant-aws + vagrant plugin install vagrant-reload + pip install --upgrade setuptools --user python + open /Applications/Docker.app + ``` +3. In your AWS console you need to reserve/create an AWS Elastic IP, a Subnet id, Security Group id, and a key pair (key pair name & *.pem file[remember to set permissions to chmod 400]). + +4. The following will clear existing Vagrant, Docker & Maven builds and deployments. WARNING - THESE STEPS WILL DISTROY ALL LOCAL DOCKER CONTAINERS AND VAGRANT BOXES. It will also create a new Vagrant box. +``` +vagrant halt node1 -f +vagrant halt default -f +vagrant destroy node1 -f +vagrant destroy default -f +for i in `vagrant global-status | grep virtualbox | awk '{print $1 }'` ; do vagrant destroy $i ; done +vagrant global-status --prune +docker rm $(docker ps -aq) +osascript -e 'quit app "Docker"' +open -a Docker +rm -rf /../.m2/repository/* +rm -rf /../.vagrant.d/boxes/* +vagrant box add dummy --force https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +``` +5. Associate your AWS ids and key information to the following enviroment variables (enter values inside the quotes). +``` +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export AWS_KEYNAME='' +export AWS_KEYPATH='../..*.pem' +export AWS_ELASTIC_IP="" +export AWS_SECURITYGROUP_ID="" +export AWS_SUBNET_ID="" +``` +6. Update your local /etc/hosts file to include AWS_ELASTIC_IP which will be tied to "node1" (do only once). +``` +sed -i "$AWS_ELASTIC_IP node1" /etc/hosts +``` + +### Deploy Metron + +1. Ensure that the Docker service is running. + +2. Deploy Metron + + ``` + cd metron-deployment/development/aws-centos6 + vagrant up --provider=aws + ``` + +### Explore Metron + +Navigate to the following resources to explore your newly minted Apache Metron environment. + +* [Metron Alerts](http://:4201) +* [Ambari](http://:8080) From ab45375977de249cc49aeb5e7f03e6c72fd0c022 Mon Sep 17 00:00:00 2001 From: as22323 Date: Wed, 4 Apr 2018 19:14:58 +0000 Subject: [PATCH 30/30] Create Vagrant --- .../0.4.1SingleNodeEC2Vagrant/Vagrant | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/Vagrant diff --git a/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/Vagrant b/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/Vagrant new file mode 100644 index 0000000000..c1d5f62112 --- /dev/null +++ b/metron-deployment/other-examples/0.4.1SingleNodeEC2Vagrant/Vagrant @@ -0,0 +1,194 @@ +#Global Cybersecurity Resource 2017-2018 (cugcr.ca) +# +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +#a distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## +#ABOUT: +#------------------- +#This Vagrant file will deploy Apache Metron as a single node instance (node1) to AWS EC2. +# +#This file was originally adapted from the Metron Single Node Install Vagrant file located here: +#https://cwiki.apache.org/confluence/display/METRON/Dev+VM+Install +#https://github.com/apache/metron/blob/master/metron-deployment/vagrant/full-dev-platform/Vagrantfile +########################################################################## +#LOCATION OF FILE: +#------------------- +#This file should be located in the following directory: +#/../metron-deployment/vagrant/full-dev-platform +########################################################################## +#REQUIREMENTS(What is needed to run this file): +#------------------- +#install: vagrant plugin install vagrant-aws +#install: vagrant plugin install vagrant-reload +#install: vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +#install: pip install --upgrade setuptools --user python +# +#You will also need to reserve an AWS Elastic IP and have a Subnet id and Security Group id +#In the code below you will see "<_comment_>" letting you know where to place the ids +########################################################################## +#BEFORE RUNNING: +#Recommended steps before running file (clearing vagrant, docker, ext..): +#------------------- +#vagrant halt node1 -f +#vagrant halt default -f +#vagrant destroy node1 -f +#vagrant destroy default -f +#for i in `vagrant global-status | grep virtualbox | awk '{print $1 }'` ; do vagrant destroy $i ; done +#vagrant global-status --prune +#docker rm $(docker ps -aq) +#osascript -e 'quit app "Docker"' +#open -a Docker +#rm -rf /../.m2/repository/* +#rm -rf /../.vagrant.d/boxes/* +#vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +########################################################################## +#HOW TO EXECUTE THIS FILE: +#------------------- +#Set Environment Variables & Run Vagrant +#------------------- +#export AWS_ACCESS_KEY_ID='' +#export AWS_SECRET_ACCESS_KEY='' +#export AWS_KEYNAME='' +#export AWS_KEYPATH='../..*.pem' +# +#export AWS_ELASTICIP="" +#export AWS_SECURITYGROUPID="" +#export AWS_SUBNETID="" +# +#vagrant up --provider=aws +########################################################################## +#OTHER NOTES: +#------------------- +#How to find Canadian Centos7 AMI ids: +#run: pip install awscli +#run: aws --region ca-central-1 ec2 describe-images --owners aws-marketplace --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce +#imageIDs: ami-28823e4c, ami-af62d0cb, ami-cbd069af + +require 'getoptlong' +require 'vagrant-aws' +require 'vagrant-reload' + +ansibleTags='' +ansibleSkipTags='sensors,quick_dev' + +begin + opts = GetoptLong.new( + [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ], + [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ] + ) + + opts.quiet = TRUE + + opts.each do |opt, arg| + case opt + when '--ansible-tags' + ansibleTags=arg + when '--ansible-skip-tags' + ansibleSkipTags=arg + end + end +rescue Exception => ignored +#Ignore to allow other opts to be passed to Vagrant +end + +puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != '' +puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != '' + +hosts = [{ + hostname: "node1", + ip: ENV['AWS_ELASTICIP'], + memory: "6000", + cpus: 4, + promisc: 2 # enables promisc on the 'Nth' network interface +}] + +Vagrant.configure(2) do |config| + config.vm.provider :aws do |aws, override| + aws.access_key_id = ENV['AWS_ACCESS_KEY_ID'] + aws.secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] + aws.keypair_name = ENV['AWS_KEYNAME'] + + #Below is a CentOS ami id available in Canada + aws.ami = "ami-00e45864" + aws.region = "ca-central-1" + aws.instance_type = "t2.2xlarge" + + #Example of a Security Group id: sg-1193xxxx + aws.security_groups = ENV['AWS_SECURITYGROUPID'] + + #Example of Subnet id: subnet-fbc3exxxx + aws.subnet_id = ENV['AWS_SUBNETID'] + aws.elastic_ip = ENV['AWS_ELASTICIP'] + override.vm.box = "dummy" + override.ssh.username = "centos" + override.ssh.private_key_path = ENV['AWS_KEYPATH'] + + #config.vm.provider "aws" do |aws, override| + override.nfs.functional = false + aws.tags = { + 'Name' => 'Phase1.5-MetronA-TESTING', + 'Organization' => 'GCR', + 'Developer' => 'GCR Dev Team', + 'Function' => 'MetronInstall' + } + aws.block_device_mapping = [{'DeviceName' =>'/dev/sda1', 'Ebs.VolumeSize' => 100, 'Ebs.DeleteOnTermination' => true}] + end + + #The following will install rsyslog, change hostname to node1, resize the disk partition, and reboot + config.vm.provision "shell", inline: <<-SHELL + yum install -y wget + yum install yum-utils + #wget http://rpms.adiscon.com/v8-stable/rsyslog.repo + #mv rsyslog.repo /etc/yum.repos.d/rsyslog.repo + #yum info rsyslog --skip-broken + #yum install -y rsyslog + yum-config-manager --add-repo http://rpms.adiscon.com/v8-stable/epel-6/x86_64 + yum install --nogpg -y rsyslog rsyslog-kafka + rm /etc/rsyslog.d/listen.conf + rsyslogd -N1 && rsyslogd + hostname node1 + sed -i "s/^HOSTNAME=.*/HOSTNAME=node1/g" /etc/sysconfig/network + echo -e "u s\nd 1\nn\np\n1\n2048\n\na\n1\nw\n" | fdisk /dev/xvda + ip link set eth0 promisc on + ifconfig eth0 promisc + #(echo u s; echo d 1; echo n; echo p; echo 1; echo 2048 ; echo ;echo a; echo 1; echo w) | fdisk {{ vol_src_1 }} || true + # + #INSTALL NIFI + #------------------- + #sudo -i + #cd /opt + #wget https://archive.apache.org/dist/nifi/1.2.0/nifi-1.2.0-bin.tar.gz + #tar xf nifi-1.2.0-bin.tar.gz + #note: modify nifi-1.1.2/conf/nifi.properties to change "nifi.web.http.port" to port 8089 + #sed -i "s/^nifi\.web\.http\.port=.*/nifi\.web\.http\.port=8089/g" /opt/nifi-1.2.0/conf/nifi.properties + #/opt/nifi-1.2.0/bin/nifi.sh install + #/opt/nifi-1.2.0/bin/nifi.sh stop + #/opt/nifi-1.2.0/bin/nifi.sh start + shutdown -r now + SHELL + + config.vm.provision :reload + + config.vm.provision :ansible do |ansible| + ansible.playbook = "../../playbooks/metron_full_install.yml" + ansible.host_key_checking = false + ansible.limit = 'all' + ansible.sudo = true + ansible.tags = ansibleTags.split(",") if ansibleTags != '' + ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' + ansible.inventory_path = "../../inventory/full-dev-platform" + end +end