Created
March 15, 2018 08:35
-
-
Save murlock/d0556c049f466dda45141102418d5c18 to your computer and use it in GitHub Desktop.
Basic script to deploy HDFS
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # this script should be launched on each node of cluster | |
| # on root account (should be updated later) | |
| # then on ${PREFIX}-hadoop-1 | |
| # go to /root/haddop | |
| # $ bin/hdfs namenode -format | |
| # $ sbin/start-dfs.sh | |
| # Check on ${PREFIX}-hadoop-1:50070 status of cluster | |
| PREFIX=${PREFIX:-customer} | |
| yum install -y wget | |
| wget http://murlock.org/hadoop-2.6.0-cdh5.11.0.tar.gz | |
| tar xf hadoop-2.6.0-cdh5.11.0.tar.gz | |
| ln -s hadoop-2.6.0-cdh5.11.0 hadoop | |
| cat >>/etc/hosts <<EOF | |
| 10.0.2.166 ${PREFIX}-storagenode-3 | |
| 10.0.2.165 ${PREFIX}-storagenode-2 | |
| 10.0.2.164 ${PREFIX}-storagenode-1 | |
| 10.0.2.167 ${PREFIX}-gwnode-1 | |
| 10.0.2.168 ${PREFIX}-hadoop-1 | |
| EOF | |
| cd ~/hadoop | |
| sed -i "s/export JAVA_HOME=\${JAVA_HOME}/export JAVA_HOME=\/usr/g" etc/hadoop/hadoop-env.sh | |
| mkdir -p /tmp/hadoop/data/namenode /tmp/hadoop/data/datanode | |
| cat > etc/hadoop/hdfs-site.xml <<EOF | |
| <?xml version="1.0" encoding="UTF-8"?> | |
| <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> | |
| <configuration> | |
| <property> | |
| <name>dfs.replication</name> | |
| <value>3</value> | |
| </property> | |
| <property> | |
| <name>dfs.namenode.name.dir</name> | |
| <value>/tmp/hadoop/data/namenode</value> | |
| </property> | |
| <property> | |
| <name>dfs.datanode.data.dir</name> | |
| <value>/tmp/hadoop/data/datanode</value> | |
| </property> | |
| </configuration> | |
| EOF | |
| mkdir -p /tmp/hadoop/dfs | |
| cat > etc/hadoop/core-site.xml <<EOF | |
| <?xml version="1.0" encoding="UTF-8"?> | |
| <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> | |
| <configuration> | |
| <property> | |
| <name>fs.defaultFS</name> | |
| <value>hdfs://${PREFIX}-hadoop-1:9000</value> | |
| </property> | |
| <property> | |
| <name>hadoop.tmp.dir</name> | |
| <value>/tmp/hadoop/dfs</value> | |
| </property> | |
| </configuration> | |
| EOF | |
| cat > etc/hadoop/slaves <<EOF | |
| ${PREFIX}-storagenode-3 | |
| ${PREFIX}-storagenode-2 | |
| ${PREFIX}-storagenode-1 | |
| EOF | |
| # don t forget to copy master key to slave, ssh must works without passwd between master to slaves |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment