Last active
April 21, 2022 11:18
-
-
Save llk23r/2d8c97d10410eff363c9828fb6e9c566 to your computer and use it in GitHub Desktop.
Revisions
-
llk23r revised this gist
Sep 26, 2017 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -40,7 +40,7 @@ exit exit sudo adduser hduser sudo su hduser sudo mkdir /usr/local/hadoop sudo cp -R $HOME/Downloads/hadoop-2.8.1/* /usr/local/hadoop sudo chown -R hduser:hadoop /usr/local/hadoop echo "===================writing to bashrc========" -
Sep 26, 2017 . 1 changed file with 3 additions and 3 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -27,12 +27,12 @@ echo "===============KEY GENERATED SUCCESSFULLY==============" cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys ssh localhost echo "===================DOWNLOADING HADOOP================" wget http://www-eu.apache.org/dist/hadoop/common/hadoop-2.8.1/hadoop-2.8.1.tar.gz clear echo "==================DOWNLOADED SUCCESSFULLY==============" clear echo "====================UNtar================" tar -xvzf hadoop-2.8.1.tar.gz clear echo "=====untar successful==============" cd / @@ -41,7 +41,7 @@ exit sudo adduser hduser sudo su hduser sudo mkdir usr/local/hadoop sudo cp -R $HOME/Downloads/hadoop-2.8.1/* /usr/local/hadoop sudo chown -R hduser:hadoop /usr/local/hadoop echo "===================writing to bashrc========" sudo cat<<EOF >>~/.bashrc -
Sep 26, 2017 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -4,7 +4,7 @@ sudo apt-get update clear echo "--------------UPDATED--------" sudo apt-get install default-jdk clear java -version echo "-------------JAVA INSTALLED----" -
Sep 26, 2017 . 1 changed file with 8 additions and 11 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -4,7 +4,7 @@ sudo apt-get update clear echo "--------------UPDATED--------" sudo apt-get install openjdk-7-jdk clear java -version echo "-------------JAVA INSTALLED----" @@ -26,14 +26,6 @@ ssh-keygen -t rsa -P "" echo "===============KEY GENERATED SUCCESSFULLY==============" cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys ssh localhost echo "===================DOWNLOADING HADOOP================" wget http://mirrors.sonic.net/apache/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz clear @@ -43,9 +35,14 @@ echo "====================UNtar================" tar -xvzf hadoop-2.7.1.tar.gz clear echo "=====untar successful==============" cd / exit exit sudo adduser hduser sudo su hduser sudo mkdir usr/local/hadoop sudo cp -R $HOME/Downloads/hadoop-2.7.1/* /usr/local/hadoop sudo chown -R hduser:hadoop /usr/local/hadoop echo "===================writing to bashrc========" sudo cat<<EOF >>~/.bashrc #HADOOP VARIABLES START -
Sep 26, 2017 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,134 @@ #!/usr/bin/env bash sudo apt-get update clear echo "--------------UPDATED--------" sudo apt-get install default-jdk clear java -version echo "-------------JAVA INSTALLED----" clear sudo addgroup hadoop echo "============HADOOP GROUP ADDED========" clear sudo adduser --ingroup hadoop hduser echo "===========HDUSER USER ADDED==========" clear sudo apt-get install ssh which ssh which sshd echo "=============SSH INSTALLED==========" clear su hduser echo "==============INSIDE HDUSER==========" ssh-keygen -t rsa -P "" echo "===============KEY GENERATED SUCCESSFULLY==============" cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys ssh localhost cd / sudo mkdir usr/local/hadoop sudo cp -R /home/mca16/Downloads/hadoop-2.8.1/* /usr/local/hadoop exit exit sudo adduser hduser sudo su hduser clear echo "===================DOWNLOADING HADOOP================" wget http://mirrors.sonic.net/apache/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz clear echo "==================DOWNLOADED SUCCESSFULLY==============" clear echo "====================UNtar================" tar -xvzf hadoop-2.7.1.tar.gz clear echo "=====untar successful==============" sudo cp -r $HOME/Downloads/hadoop-2.7.1/* /usr/local/hadoop/ sudo chown -R hduser:hadoop /usr/local/hadoop clear echo "===================writing to bashrc========" sudo cat<<EOF >>~/.bashrc #HADOOP VARIABLES START export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 export HADOOP_INSTALL=/usr/local/hadoop export PATH=$PATH:$HADOOP_INSTALL/bin export PATH=$PATH:$HADOOP_INSTALL/sbin export HADOOP_MAPRED_HOME=$HADOOP_INSTALL export HADOOP_COMMON_HOME=$HADOOP_INSTALL export HADOOP_HDFS_HOME=$HADOOP_INSTALL export YARN_HOME=$HADOOP_INSTALL export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_INSTALL/lib/native export HADOOP_OPTS="-Djava.library.path=$HADOOP_INSTALL/lib" #HADOOP VARIABLES END EOF sudo cat<<EOF >>/usr/local/hadoop/etc/hadoop/hadoop-env.sh export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 EOF sudo mkdir -p /app/hadoop/tmp sudo chown hduser:hadoop /app/hadoop/tmp sudo cat<<EOF >>/usr/local/hadoop/etc/hadoop/core-site.xml <configuration> <property> <name>hadoop.tmp.dir</name> <value>/app/hadoop/tmp</value> <description>A base for other temporary directories.</description> </property> <property> <name>fs.default.name</name> <value>hdfs://localhost:54310</value> <description>The name of the default file system. A URI whose scheme and authority determine the FileSystem implementation. The uri's scheme determines the config property (fs.SCHEME.impl) naming the FileSystem implementation class. The uri's authority is used to determine the host, port, etc. for a filesystem.</description> </property> </configuration> EOF sudo cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml cat<<EOF >>/usr/local/hadoop/etc/hadoop/mapred-site.xml <property> <name>mapred.job.tracker</name> <value>localhost:54311</value> <description>The host and port that the MapReduce job tracker runs at. If "local", then jobs are run in-process as a single map and reduce task. </description> </property> EOF sudo mkdir -p /usr/local/hadoop_store/hdfs/namenode sudo mkdir -p /usr/local/hadoop_store/hdfs/datanode sudo chown -R hduser:hadoop /usr/local/hadoop_store cat<<EOF >>/usr/local/hadoop/etc/hadoop/hdfs-site.xml <property> <name>dfs.replication</name> <value>1</value> <description>Default block replication. The actual number of replications can be specified when the file is created. The default is used if replication is not specified in create time. </description> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/usr/local/hadoop_store/hdfs/namenode</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/usr/local/hadoop_store/hdfs/datanode</value> </property> EOF hadoop namenode -format start-dfs.sh jps