This repository has been archived by the owner on Jan 3, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 19
/
centosvmCDP.sh
203 lines (162 loc) · 6.87 KB
/
centosvmCDP.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
#! /bin/bash
echo "-- Configure user cloudera with passwordless and pem file"
useradd cloudera -d /home/cloudera -p cloudera
sudo usermod -aG wheel cloudera
cp /etc/sudoers /etc/sudoers.bkp
rm -rf /etc/sudoers
sed '/^#includedir.*/a cloudera ALL=(ALL) NOPASSWD: ALL' /etc/sudoers.bkp > /etc/sudoers
ssh-keygen -t rsa -b 2048 -P '' -f ~/.ssh/clouderakey
cd /home/vagrant/.ssh/
cat clouderakey.pub >> authorized_keys
chmod 0600 authorized_keys
openssl rsa -in clouderakey -outform pem > ~/clouderakey.pem
chmod 0400 ~/clouderakey.pem
cp ~/cloudera.pem /home/cloudera
cp ~/cloudera.pem /home/vagrant
cp -r /home/vagrant/.ssh /home/cloudera/.ssh
cd ~
echo "-- Configure and optimize the OS"
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local
# add tuned optimization https://www.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_admin_performance.html
echo "vm.swappiness = 1" >> /etc/sysctl.conf
sysctl vm.swappiness=1
timedatectl set-timezone UTC
echo "-- Install Java OpenJDK8 and other tools"
yum install -y java-1.8.0-openjdk-devel vim wget curl git bind-utils rng-tools
yum install -y epel-release
yum install -y python-pip
cp /usr/lib/systemd/system/rngd.service /etc/systemd/system/
systemctl daemon-reload
systemctl start rngd
systemctl enable rngd
echo "server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4" >> /etc/chrony.conf
systemctl restart chronyd
sudo /etc/init.d/network restart
echo "-- Configure networking"
#PUBLIC_IP=`curl https://api.ipify.org/`
#hostnamectl set-hostname `hostname -f`
#sed -i$(date +%s).bak '/^[^#]*cloudera/s/^/# /' /etc/hosts
sed -i$(date +%s).bak '/^[^#]*::1/s/^/# /' /etc/hosts
#sed -i$(date +%s).bak 's/127\.0\.0\.1/& cloudera /' /etc/hosts
#echo "`host localhost |grep address | awk '{print $4}'` `hostname` `hostname`" >> /etc/hosts
#sed -i "s/HOSTNAME=.*/HOSTNAME=`hostname`/" /etc/sysconfig/network
systemctl disable firewalld
systemctl stop firewalld
service firewalld stop
setenforce 0
sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
echo "Disabling IPv6"
echo "net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv6.conf.eth0.disable_ipv6 = 1" >> /etc/sysctl.conf
sysctl -p
sudo service network-manager restart
echo "-- Install CM and MariaDB"
# CM 7
wget https://archive.cloudera.com/cm7/7.1.4/redhat7/yum/cloudera-manager-trial.repo -P /etc/yum.repos.d/
# MariaDB 10.1
cat - >/etc/yum.repos.d/MariaDB.repo <<EOF
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.1/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1
EOF
yum clean all
rm -rf /var/cache/yum/
yum repolist
## CM
yum install -y cloudera-manager-agent cloudera-manager-daemons cloudera-manager-server
#sed -i$(date +%s).bak '/^[^#]*server_host/s/^/# /' /etc/cloudera-scm-agent/config.ini
#sed -i$(date +%s).bak '/^[^#]*listening_ip/s/^/# /' /etc/cloudera-scm-agent/config.ini
#sed -i$(date +%s).bak "/^# server_host.*/i server_host=localhost" /etc/cloudera-scm-agent/config.ini
#sed -i$(date +%s).bak "/^# listening_ip=.*/i listening_ip=127.0.0.1" /etc/cloudera-scm-agent/config.ini
service cloudera-scm-agent restart
## MariaDB
yum install -y MariaDB-server MariaDB-client
cat /root/CDPDCTrial/conf/mariadb.config > /etc/my.cnf
echo "--Enable and start MariaDB"
systemctl enable mariadb
systemctl start mariadb
echo "-- Install JDBC connector"
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz -P ~
tar zxf ~/mysql-connector-java-5.1.46.tar.gz -C ~
mkdir -p /usr/share/java/
cp ~/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
rm -rf ~/mysql-connector-java-5.1.46*
echo "-- Create DBs required by CM"
mysql -u root < /root/CDPDCTrial/scripts/create_db.sql
echo "-- Secure MariaDB"
mysql -u root < /root/CDPDCTrial/scripts/secure_mariadb.sql
echo "-- Prepare CM database 'scm'"
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm cloudera
## PostgreSQL
#yum install -y postgresql-server python-pip
#pip install psycopg2==2.7.5 --ignore-installed
#echo 'LC_ALL="en_US.UTF-8"' >> /etc/locale.conf
#sudo su -l postgres -c "postgresql-setup initdb"
#cat conf/pg_hba.conf > /var/lib/pgsql/data/pg_hba.conf
#cat conf/postgresql.conf > /var/lib/pgsql/data/postgresql.conf
#echo "--Enable and start pgsql"
#systemctl enable postgresql
#systemctl restart postgresql
## PostgreSQL see: https://www.postgresql.org/download/linux/redhat/
yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
yum install -y postgresql96
yum install -y postgresql96-server
pip install psycopg2==2.7.5 --ignore-installed
echo 'LC_ALL="en_US.UTF-8"' >> /etc/locale.conf
/usr/pgsql-9.6/bin/postgresql96-setup initdb
cat /root/CDPDCTrial/conf/pg_hba.conf > /var/lib/pgsql/9.6/data/pg_hba.conf
cat /root/CDPDCTrial/conf/postgresql.conf > /var/lib/pgsql/9.6/data/postgresql.conf
echo "--Enable and start pgsql"
systemctl enable postgresql-9.6
systemctl start postgresql-9.6
echo "-- Create DBs required by CM"
sudo -u postgres psql <<EOF
CREATE DATABASE ranger;
CREATE USER ranger WITH PASSWORD 'cloudera';
GRANT ALL PRIVILEGES ON DATABASE ranger TO ranger;
CREATE DATABASE das;
CREATE USER das WITH PASSWORD 'cloudera';
GRANT ALL PRIVILEGES ON DATABASE das TO das;
EOF
echo "-- Enable passwordless root login via rsa key"
ssh-keygen -f ~/myRSAkey -t rsa -N ""
mkdir ~/.ssh
cat ~/myRSAkey.pub >> ~/.ssh/authorized_keys
chmod 400 ~/.ssh/authorized_keys
ssh-keyscan -H `hostname` >> ~/.ssh/known_hosts
sed -i 's/.*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config
systemctl restart sshd
echo "-- Prepare parcels dirs"
echo "-- Start CM, it takes about 2 minutes to be ready"
systemctl start cloudera-scm-server
while [ `curl -s -X GET -u "admin:admin" http://localhost:7180/api/version` -z ] ;
do
echo "waiting 10s for CM to come up..";
sleep 10;
done
chmod -R 777 /opt/cloudera/parcel-repo
mkdir /opt/cloudera/parcel-cache
chmod -R 777 /opt/cloudera/parcel-cache
chmod -R 777 /opt/cloudera/csd
mkdir /opt/cloudera/parcels
chmod -R 777 /opt/cloudera/parcels
echo "-- Now CM is started and the next step is to automate using the CM API"
pip install cm_client
sed -i "s/YourHostname/localhost.localdomain/g" ~/CDPDCTrial/scripts/create_cluster.py
mkdir /data
mkdir /data/dfs
chmod -R 777 /data
python ~/CDPDCTrial/scripts/create_cluster.py ~/CDPDCTrial/conf/cdpsandbox.json
sudo usermod cloudera -G hadoop
sudo -u hdfs hdfs dfs -mkdir /user/cloudera
sudo -u hdfs hdfs dfs -chown cloudera:hadoop /user/cloudera
sudo -u hdfs hdfs dfs -mkdir /user/admin
sudo -u hdfs hdfs dfs -chown admin:hadoop /user/admin
sudo -u hdfs hdfs dfs -chmod -R 0755 /tmp