Last active
November 6, 2015 17:25
-
-
Save bartekdobija/251c2feabec310e2da72 to your computer and use it in GitHub Desktop.
Revisions
-
bartekdobija revised this gist
Nov 6, 2015 . 1 changed file with 13 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,3 +1,4 @@ # Basic OS configuration $sysconfig = <<SCRIPT @@ -163,6 +164,18 @@ $httpd = <<SCRIPT echo "Apache HTTPd installation..." && yum install -y httpd echo "silk.corp.ryanair.com virtual host configuration" cat <<HTTPDCNF > /etc/httpd/conf.d/silk.conf ProxyRequests On <VirtualHost *:*> ProxyPreserveHost On ProxyPass "/" "http://127.0.0.1:5601/" ProxyPassReverse "/" "http://127.0.0.1:5601/" ServerName silk.corp.ryanair.com </VirtualHost> HTTPDCNF chkconfig httpd on && service httpd start SCRIPT -
Nov 6, 2015 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,292 @@ # Basic OS configuration $sysconfig = <<SCRIPT # disable IPv6 echo "net.ipv6.conf.all.disable_ipv6=1" > /etc/sysctl.conf && sysctl -f /etc/sysctl.conf # this should be a persistent config ulimit -n 65536 ulimit -s 10240 ulimit -c unlimited # detect ryanair network PROXY_CONFIG=/etc/profile.d/proxy.sh if grep ryanair /etc/resolv.conf; then echo "export http_proxy=http://internalproxy.corp.ryanair.com:3128" > ${PROXY_CONFIG} \ && echo "export https_proxy=http://internalproxy.corp.ryanair.com:3128" >> ${PROXY_CONFIG} \ && echo "export no_proxy=localhost,127.0.0.0/8,127.0.1.1,127.0.1.1*,local.home" >> ${PROXY_CONFIG} else rm -fR ${PROXY_CONFIG} fi service iptables stop && chkconfig iptables off # Add entries to /etc/hosts ip=$(ifconfig eth1 | awk -v host=$(hostname) '/inet addr/ {print substr($2,6)}') host=$(hostname) echo "127.0.0.1 localhost" > /etc/hosts echo "$ip $host" >> /etc/hosts PASSWORD=$(echo "$(date)$RANDOM" | md5sum | awk '{print $1}') USERS="kafka zookeeper solr" for u in $USERS; do if ! grep ${u} /etc/passwd; then echo "Creating user ${u}" \ && useradd -p $(openssl passwd -1 ${PASSWORD}) ${u} fi done SCRIPT $javajdk = <<SCRIPT yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel git lsof SCRIPT # Zookeeper installation & configuration $zookeeper = <<SCRIPT ZOOKPR_LINK=/opt/zookeeper if [ ! -e ${ZOOKPR_LINK} ]; then echo "Zookeeper installation..." ZOOKPR_VER=zookeeper-3.4.6 ZOOKPR_DAT=/opt/zookeeper-data wget http://mirrors.whoishostingthis.com/apache/zookeeper/zookeeper-3.4.6/${ZOOKPR_VER}.tar.gz -q -P /tmp/ \ && tar zxf /tmp/${ZOOKPR_VER}.tar.gz -C /opt/ \ && ln -f -s /opt/${ZOOKPR_VER} ${ZOOKPR_LINK} \ && echo "export ZOO_LOG_DIR=/opt/zookeeper/logs" > /etc/profile.d/zookeeper.sh \ && echo "export PATH=\\${PATH}:${ZOOKPR_LINK}/bin" >> /etc/profile.d/zookeeper.sh \ && mkdir -p ${ZOOKPR_DAT} \ && mkdir -p ${ZOOKPR_LINK}/logs \ && chown -R zookeeper:zookeeper ${ZOOKPR_DAT} \ && chown -R zookeeper:zookeeper /opt/${ZOOKPR_VER} \ && echo "dataDir=${ZOOKPR_DAT}" > ${ZOOKPR_LINK}/conf/zoo.cfg \ && echo "maxClientCnxns=0" >> ${ZOOKPR_LINK}/conf/zoo.cfg \ && echo "clientPort=2181" >> ${ZOOKPR_LINK}/conf/zoo.cfg fi cat << ZOOINITD > /etc/init.d/zookeeper #! /bin/sh # /etc/init.d/zookeeper: start the zookeeper daemon. # chkconfig: 345 98 01 # description: zookeeper ZOOKEEPER_HOME=/opt/zookeeper ZOOKEEPER_CONFIG=\\${ZOOKEEPER_HOME}/conf/server.properties USER=zookeeper function start { sudo -E -u \\${USER} ZOO_LOG_DIR=${ZOOKPR_LINK}/logs \ JVMFLAGS="-Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" \ \\${ZOOKEEPER_HOME}/bin/zkServer.sh start } function stop { sudo -E -u \\${USER} \\${ZOOKEEPER_HOME}/bin/zkServer.sh stop } case "\\$1" in start) start ;; stop) stop ;; restart) stop start ;; *) echo "Usage: \\$0 {start|stop|restart}" exit 1 esac ZOOINITD chmod 755 /etc/init.d/zookeeper \ && chkconfig zookeeper on if [ ! "$(ps aux | grep /opt/zookeeper | wc -l)" == "2" ]; then service zookeeper start fi SCRIPT $nodejs = <<SCRIPT NODEJS_LINK=/opt/node NODEJS_VERSION=v4.2.2 if [ ! -e ${NODEJS_LINK} ]; then echo "Node.js installation..." wget https://nodejs.org/dist/${NODEJS_VERSION}/node-${NODEJS_VERSION}-linux-x64.tar.gz -q -P /tmp/ \ && tar zxf /tmp/node-${NODEJS_VERSION}-linux-x64.tar.gz -C /opt/ \ && ln -f -s /opt/node-${NODEJS_VERSION}-linux-x64 ${NODEJS_LINK} \ && echo "PATH=\\${PATH}:${NODEJS_LINK}/bin" > /etc/profile.d/nodejs.sh \ && ${NODEJS_LINK}/bin/npm install -g bower \ && ${NODEJS_LINK}/bin/npm install -g grunt \ && ${NODEJS_LINK}/bin/npm install -g grunt-cli fi SCRIPT $silk = <<SCRIPT SOLR_LINK=/opt/solr SOLR_VERSION=5.3.1 if [ ! -e ${SOLR_LINK} ]; then echo "Solr installation..." wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz -q -P /tmp/ \ && tar zxf /tmp/solr-${SOLR_VERSION}.tgz -C /opt/ \ && ln -f -s /opt/solr-${SOLR_VERSION} ${SOLR_LINK} \ && echo "PATH=\\${PATH}:${SOLR_LINK}/bin" > /etc/profile.d/solr.sh \ && chown -R solr:solr /opt/solr-${SOLR_VERSION} \ && ${SOLR_LINK}/bin/solr start -c > /dev/null fi SILK_LINK=/opt/silk if [ ! -e ${SILK_LINK} ]; then rm -fR /tmp/silk && rm -fR /opt/silk-0 \ && git clone https://github.com/bartekdobija/silk.git /opt/silk-0 \ && ln -f -s /opt/silk-0 ${SILK_LINK} \ && > ${SILK_LINK}/.git/hooks/pre-commit \ && > ${SILK_LINK}/.git/hooks/pre-push \ && > ${SILK_LINK}/.git/hooks/post-merge \ && cd ${SILK_LINK} \ && npm install \ && bower --allow-root install \ && grunt build \ && ${SOLR_LINK}/bin/solr create -c silkconfig -d ${SILK_LINK}/silkconfig/ fi SCRIPT $httpd = <<SCRIPT echo "Apache HTTPd installation..." && yum install -y httpd chkconfig httpd on && service httpd start SCRIPT # Kafka installation & configuration $kafka = <<SCRIPT KAFKA_LINK=/opt/kafka if [ ! -e ${KAFKA_LINK} ]; then echo "Kafka installation..." KAFKA_VER=kafka_2.11-0.8.2.2 wget http://mirrors.whoishostingthis.com/apache/kafka/0.8.2.2/${KAFKA_VER}.tgz -q -P /tmp/ \ && tar zxf /tmp/${KAFKA_VER}.tgz -C /opt/ \ && ln -f -s /opt/${KAFKA_VER} ${KAFKA_LINK} \ && echo "PATH=\\${PATH}:${KAFKA_LINK}/bin" > /etc/profile.d/kafka.sh \ && mkdir -p ${KAFKA_LINK}/logs \ && chown -R kafka:kafka /opt/${KAFKA_VER} fi cat << KFINITD > /etc/init.d/kafka #! /bin/sh # /etc/init.d/kafka: start the kafka daemon. # chkconfig: 345 99 01 # description: kafka KAFKA_HOME=/opt/kafka KAFKA_CONFIG=\\${KAFKA_HOME}/config/server.properties USER=kafka KAFKA_JMX_PORT=9666 function start { sudo -E -u \\${USER} JMX_PORT=\\${KAFKA_JMX_PORT} \\${KAFKA_HOME}/bin/kafka-server-start.sh -daemon \\${KAFKA_CONFIG} } function stop { sudo -u \\${USER} \\${KAFKA_HOME}/bin/kafka-server-stop.sh } case "\\$1" in start) start ;; stop) stop ;; restart) stop start ;; *) echo "Usage: \\$0 {start|stop|restart}" exit 1 esac KFINITD chmod 755 /etc/init.d/kafka \ && chkconfig kafka on if [ ! "$(ps aux | grep /opt/kafka | wc -l)" == "2" ]; then service kafka start fi # create sample topics for topic in clickstream flightinfo transactions ryanaircom myryanair; do ${KAFKA_LINK}/bin/kafka-topics.sh \ --create \ --zookeeper localhost:2181 \ --topic ${topic} \ --partitions 1 \ --replication-factor 1 &> /dev/null done SCRIPT # Info $information = <<SCRIPT ip=$(ifconfig eth1 | awk -v host=$(hostname) '/inet addr/ {print substr($2,6)}') echo "" echo "VM's IP address: $ip" echo "" echo "Start Zookeeper with the below command:" echo "sudo service zookeeper start" echo "logs avail. under /opt/zookeeper/logs" echo "JMX Endpoint available at: $ip:9999" echo "" echo "Start Kafka with the below command:" echo "sudo service kafka start" echo "logs avail. under /opt/kafka/logs" echo "JMX Endpoint available at: $ip:9666" echo "--> Kafka broker available at: $ip:9092 <--" echo "" echo "Available (sample) Kafka topics:" kafka-topics.sh --list --zookeeper localhost:2181 echo "" echo "Test commands:" echo "kafka-topics.sh --zookeeper localhost:2181 --list" echo "kafka-topics.sh --zookeeper localhost:2181 --create --topic <topic_name> --partitions 1 --replication-factor 1" echo "kafka-console-consumer.sh --zookeeper localhost:2181 --blacklist none" echo "kafka-console-producer.sh --broker-list $ip:9092 --topic flightinfo" echo "> test event" echo " " SCRIPT Vagrant.configure(2) do |config| config.vm.box = "boxcutter/centos66" config.vm.hostname = "kafka.instance.com" config.vm.network :public_network, :mac => "0800DEADBEEF" config.vm.provider "virtualbox" do |vb| vb.name = "kafka-node" vb.cpus = 2 vb.memory = 4096 vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] vb.customize ["modifyvm", :id, "--cpuexecutioncap", "100"] end config.vm.provision :shell, :name => "sysconfig", :inline => $sysconfig config.vm.provision :shell, :name => "javajdk", :inline => $javajdk config.vm.provision :shell, :name => "zookeeper", :inline => $zookeeper config.vm.provision :shell, :name => "kafka", :inline => $kafka config.vm.provision :shell, :name => "httpd", :inline => $httpd config.vm.provision :shell, :name => "nodejs", :inline => $nodejs config.vm.provision :shell, :name => "silk", :inline => $silk config.vm.provision :shell, :name => "information", :inline => $information end