Last active
July 23, 2018 08:19
-
-
Save Dextaa/3f4461733c49d80ec7cc96a8f26aa317 to your computer and use it in GitHub Desktop.
Revisions
-
Dextaa revised this gist
Nov 20, 2016 . 1 changed file with 3 additions and 3 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -98,7 +98,7 @@ start_zookeeper() { echo "export SERVER_JVMFLAGS=-DXmx2G" >> /home/ec2-user/.bash_profile # Schedule starting the ZK server to ensure that the instance is up and all configuration has run # The time based scheduling could obviously be made more sophisticated should it be required. runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$ZK_VERSION/bin/zkServer.sh start /home/ec2-user/$ZK_CONFIG_FILE &' | at now + 3 minute" } @@ -151,8 +151,8 @@ install_kafka() { start_kafka() { # Schedule starting the ZK server to ensure that the instance is up and all configuration has run. # The time based scheduling could obviously be made more sophisticated should it be required. runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$KAFKA_VERSION/bin/kafka-server-start.sh /home/ec2-user/$KAFKA_CONFIG_FILE &' | at now + 4 minute" } -
Nov 20, 2016 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,167 @@ #!/bin/bash # Use whichever region you're working in export AWS_DEFAULT_REGION=eu-west-1 # Make ec2-user's home dir the install to make it easier to navigate when subsequently ssh-ing in to instances su ec2-user - cd /home/ec2-user echo Working dir is `pwd` # Use the instance metadata to find out about the network the instance has been lauched into (the IP is the standard AWS metadata IP) AWS_METADATA_IP=169.254.169.254 MAC_ADDRESS=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/` SUBNET_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/${MAC_ADDRESS}subnet-id` SELF_INSTANCE_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/instance-id` S3_CONFIG_BUCKET=com.example.event-servers ENI_NAME=event-servers-eni ENI_INDEX=1 ZK_VERSION=zookeeper-3.4.9 ZK_CONFIG_FILE=zookeeper.config KAFKA_VERSION=kafka_2.11-0.10.0.1 KAFKA_CONFIG_FILE=kafka.config attach_eni() { # Use the AWS CLI to see if the target ENI is already attached.. CURRENT_ATTACHMENT_ID=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --query "NetworkInterfaces[0].[Attachment][0].[AttachmentId]" | grep -o 'eni-attach-[a-z0-9]*'` # ..and if it is, remove the attachment (the assumption here is that it's still attached to another instance that is being terminated). if [ $CURRENT_ATTACHMENT_ID"X" != "X" ]; then echo Detaching $CURRENT_ATTACHMENT_ID; aws ec2 detach-network-interface --attachment-id $CURRENT_ATTACHMENT_ID fi # Use the AWS CLI to get the id of the ENI to be attached NETWORK_INTERFACE_ID=`aws ec2 describe-network-interfaces --filters "Name=status,Values=available" "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].NetworkInterfaceId" | grep -o 'eni-[a-z0-9]*'` # Attach the ENI (and display the attachment id) echo Attaching $ENI_NAME to $ENI_INDEX `aws ec2 attach-network-interface --network-interface-id $NETWORK_INTERFACE_ID --instance-id $SELF_INSTANCE_ID --device-index $ENI_INDEX` } install_zookeeper() { # Copy the ZK config file from S3 aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_CONFIG_FILE $ZK_CONFIG_FILE # Format one of the EBS volumes associated with the instance and mount it for use as the ZK data directory. mkfs -t ext4 /dev/sdb DATA_DIR=`cat $ZK_CONFIG_FILE | grep 'dataDir=[a-z0-9/-]*' | sed 's/dataDir=//'` # This assumes that the dataDir value in the ZK config has a parent directory. The volume is mounted at the parent rather than the 'dataDir' value because the # ext4 file system adds a 'lost+found' directory at the mount point and this confuses ZK. DATA_PARENT=`dirname "${DATA_DIR}"` mkdir -p $DATA_PARENT mount /dev/sdb $DATA_PARENT mkdir -p $DATA_DIR chmod a+rwx $DATA_DIR # As per the dataDir mkfs -t ext4 /dev/sdc LOG_DIR=`cat $ZK_CONFIG_FILE | grep 'dataLogDir=[a-z0-9/-]*' | sed 's/dataLogDir=//'` LOG_PARENT=`dirname "${LOG_DIR}"` mkdir -p $LOG_PARENT mount /dev/sdc $LOG_PARENT mkdir -p $LOG_DIR chmod a+rwx $LOG_DIR # Find the 'server.<number>' entry in the ZK config whose IP address matches the ENI's address, extract the <number> part and use it as the ZK server's id ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'` MATCHING_SERVER_ID=`cat $ZK_CONFIG_FILE | sed -n "s/^server.\([0-9]*\)=$ENI_PRIVATE_IP[\:0-9]*/\1/p"` echo $MATCHING_SERVER_ID >> $DATA_DIR/myid # Use the ENI's IP address as ZK's client connection listen address echo >> $ZK_CONFIG_FILE echo "clientPortAddress=$ENI_PRIVATE_IP" >> $ZK_CONFIG_FILE # Get the ZK zip and install it ZK_INSTALL_FILE=$ZK_VERSION.tar.gz aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_INSTALL_FILE $ZK_INSTALL_FILE tar -xzf $ZK_INSTALL_FILE rm $ZK_INSTALL_FILE chown -R ec2-user:ec2-user $DATA_DIR chown -R ec2-user:ec2-user $LOG_DIR } start_zookeeper() { # Optional echo "export JMXDISABLE=true" >> /home/ec2-user/.bash_profile echo "export SERVER_JVMFLAGS=-DXmx2G" >> /home/ec2-user/.bash_profile # Schedule starting the ZK server to ensure that the instance is up and all configuration has run echo runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$ZK_VERSION/bin/zkServer.sh start /home/ec2-user/$ZK_CONFIG_FILE &' | at now + 3 minute" runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$ZK_VERSION/bin/zkServer.sh start /home/ec2-user/$ZK_CONFIG_FILE &' | at now + 3 minute" } install_kafka() { # See comments in the 'install_zookeeper' function aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_CONFIG_FILE $KAFKA_CONFIG_FILE mkfs -t ext4 /dev/sdd LOG_DIR=`cat $KAFKA_CONFIG_FILE | grep 'log\.dir=[a-z0-9/-]*' | sed 's/log\.dir=//'` LOG_PARENT=`dirname "${LOG_DIR}"` mkdir -p $LOG_PARENT mount /dev/sdd $LOG_PARENT mkdir -p $LOG_DIR chmod a+rwx $LOG_DIR ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'` # Use the last octet of the ENI's IP address as the Kafka broker's id (21, 37 or 53 are used in the associated blog post) BROKER_ID=`echo $ENI_PRIVATE_IP | awk --field-separator \. '{ print $4 }'` echo broker.id=$BROKER_ID >> $KAFKA_CONFIG_FILE echo >> $KAFKA_CONFIG_FILE # Use the ENI's IP address as the Kafka listener and advertised listener values echo listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE echo >> $KAFKA_CONFIG_FILE echo advertised.listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE echo >> $KAFKA_CONFIG_FILE # Get the Kafka zip and install it KAFKA_INSTALL_FILE=$KAFKA_VERSION.tgz aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_INSTALL_FILE $KAFKA_INSTALL_FILE tar -xzf $KAFKA_INSTALL_FILE rm $KAFKA_INSTALL_FILE chown -R ec2-user:ec2-user $LOG_DIR PRIMARY_IP=`curl http://$AWS_METADATA_IP/latest/meta-data/local-ipv4` HOST_NAME=`curl http://$AWS_METADATA_IP/latest/meta-data/local-hostname` SHORT_HOST_NAME=`echo $HOST_NAME | awk --field-separator \. '{ print $1 }'` # The default host name for an EC2 instance causes Kafka to throw an exception so create an appropriate hosts file entry which Kafka can use echo >> etc/hosts echo $PRIMARY_IP $SHORT_HOST_NAME $HOST_NAME >> /etc/hosts } start_kafka() { # Schedule starting the ZK server to ensure that the instance is up and all configuration has run echo runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$KAFKA_VERSION/bin/kafka-server-start.sh /home/ec2-user/$KAFKA_CONFIG_FILE &' | at now + 4 minute" runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$KAFKA_VERSION/bin/kafka-server-start.sh /home/ec2-user/$KAFKA_CONFIG_FILE &' | at now + 4 minute" } attach_eni install_zookeeper install_kafka chown -R ec2-user:ec2-user . start_zookeeper start_kafka