Skip to content

Instantly share code, notes, and snippets.

@ken5scal

ken5scal/ECS DEPLOYMENT

Created December 31, 2016 04:30
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ken5scal/156f4ce2565dfb993c7061d3568aa3f4 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ken5scal/156f4ce2565dfb993c7061d3568aa3f4 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. ken5scal created this gist Dec 31, 2016.
    211 changes: 211 additions & 0 deletions ECS DEPLOYMENT
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,211 @@
    #!/usr/bin/env bash
    ###
    # Updates a pre-configured Amazon EC2 Container Service (ECS) cluster.
    #
    # Build a docker image in CWD, tag it and push it to hub.docker.com.
    # Then Update an ECS services to run that newly available images.
    #
    # This script receives as a input a filename (Relative to this directory).
    #
    # That file must set the following Environment variables .
    # DOCKER_USER : Docker username for hub.docker.com
    # DOCKER_PASSWORD : Docker password for hub.docker.com
    # DOCKER_EMAIL : The email address used for registration with hub.docker.com
    # DOCKER_IMAGE_TAG : The Docker image & tag we want to build & deploy Example : "daiglej/repo:MyTag"
    # ECS_CLUSTER : The Name of the ECS cluster
    # ECS_SERVICE : The name of the ECS service (Must exists within the cluster)
    # ECS_FAMILY : Also know as the task name
    # ECS_TASK_DEFINITION : A json string that holds the Task definition.
    #
    # NOTE : ECS Deploys strategy requires that the cluster is able to run one task during an update of the task.
    # We assume that your cluster is not configured to be able to have 1 extra task running during the deploy.
    # Therefore, we adopted the strategy of decreasing by 1 the number of running task before we trigger the update.
    # If you are only running 1 instance of the task, this means you will have a DOWNTIME of approximately 30-60 seconds.
    # Therefore, unless thats not an issue, we reccomend you have 2 instances of the task.
    #
    # This script was inspired/based on : https://github.com/circleci/circle-ecs/blob/master/deploy.sh
    ###

    set -e
    set -u
    set -o pipefail
    #set -o xtrace # Uncomment for debugging
    JQ="jq --raw-output --exit-status"
    JQ="jq --raw-output" # Comment for debugging

    ##
    # Main function
    ##
    function main() {
    echo -e "\n\n$(date "+%Y-%m-%d %H:%M:%S") Executing $1"
    my_dir="$(dirname "$0")"
    source "$my_dir/$1"

    echo "$(date "+%Y-%m-%d %H:%M:%S") Building, tagging & pushing docker image ($DOCKER_IMAGE_TAG)"
    docker_build_and_push "$DOCKER_IMAGE_TAG";
    echo -e "\n\n"

    get_ecs_status;
    DESIRED_COUNT=$CURRENT_DESIRED_COUNT

    if [[ $DESIRED_COUNT>0 ]]; then
    echo "$(date "+%Y-%m-%d %H:%M:%S") Decrease the desired numberof running task instances by one ($DESIRED_COUNT - 1 =$(expr $DESIRED_COUNT - 1))"
    echo "Otherwise, the deploy will fail if cluster is not able to support one additional instance (We assume this is not the case)."
    update_ecs_service $CURRENT_TASK_REVISION $(expr $DESIRED_COUNT - 1)
    else
    echo -e "$(date "+%Y-%m-%d %H:%M:%S") Service has currently 0 desired running instances. Setting the desired running task instance to 1"
    DESIRED_COUNT=1
    fi

    echo "$(date "+%Y-%m-%d %H:%M:%S") Update the Task definition (Includes the new docker images to use)"
    revision=$(update_ecs_task_def "$ECS_TASK_DEFINITION")

    echo "$(date "+%Y-%m-%d %H:%M:%S") Update the service to use the newly created task revision ($CURRENT_TASK_REVISION)"
    update_ecs_service "$CURRENT_TASK_REVISION" "$(expr $DESIRED_COUNT - 1)"

    echo "$(date "+%Y-%m-%d %H:%M:%S") Waiting for the number of running task instance to decrease to $(expr $DESIRED_COUNT - 1)"
    wait_ecs_nb_task $(expr $DESIRED_COUNT - 1)

    echo "$(date "+%Y-%m-%d %H:%M:%S") Done ... Now we can now re-set the original desired number task instance ($DESIRED_COUNT)"
    update_ecs_service "$CURRENT_TASK_REVISION" "$DESIRED_COUNT"

    echo "$(date "+%Y-%m-%d %H:%M:%S") Waiting for the number of running task to reach the original desired number of instances ($DESIRED_COUNT)"
    wait_ecs_nb_task $DESIRED_COUNT

    echo "$(date "+%Y-%m-%d %H:%M:%S") Waiting for stale task to be replaced by their new revision"
    wait_ecs_no_stale_task

    echo "$(date "+%Y-%m-%d %H:%M:%S") Deploy completed successfully. "
    echo "THANK YOU COME AGAIN!"
    }

    ##
    # Log in to hub.docker.com, build the docker image, tag it and push it to hub.docker.com
    #
    # @params string $1 The Name of the docker image & tag Example daiglej/repo:TAG
    # @reads string $DOCKER_EMAIL hub.docker.com registration email
    # @reads string $DOCKER_PASSWORD hub.docker.com password
    # @reads string $DOCKER_USER hub.docker.com registration username
    ##
    function docker_build_and_push() {
    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" -e "$DOCKER_EMAIL"
    docker build -t $1 .
    docker push $1
    }

    ##
    # Read various status info about the ECS services and set/update status variables accordingly.
    #
    # @reads string $ECS_CLUSTER The name of the ECS cluster
    # @reads string $ECS_SERVICE The name of the ECS service (Inside the cluster)
    # @reads string $AWS_ACCESS_KEY_ID AWS Access key
    # @reads string $AWS_DEFAULT_REGION AWS Default Region
    # @reads string $AWS_SECRET_ACCESS_KEY AWS Secret access key
    # @sets int $CURRENT_DESIRED_COUNT The number of tasks that should be running
    # @sets int $CURRENT_RUNNING_TASK The number of task that are running
    # @sets int $CURRENT_STALE_TASK The number of running task, that are not of the current revision
    # @sets string $CURRENT_TASK_REVISION The current task revision/version (Full ARN)
    ##
    function get_ecs_status() {
    DECRIBED_SERVICE=$(aws ecs describe-services --cluster $ECS_CLUSTER \
    --services $ECS_SERVICE);

    CURRENT_DESIRED_COUNT=$(echo $DECRIBED_SERVICE | $JQ ".services[0].desiredCount")
    CURRENT_TASK_REVISION=$(echo $DECRIBED_SERVICE | $JQ ".services[0].taskDefinition")
    CURRENT_RUNNING_TASK=$(echo $DECRIBED_SERVICE | $JQ ".services[0].runningCount")
    CURRENT_STALE_TASK=$(echo $DECRIBED_SERVICE | $JQ ".services[0].deployments | .[] | select(.taskDefinition != \"$CURRENT_TASK_REVISION\") | .taskDefinition")
    if [[ -z "$CURRENT_STALE_TASK" ]]; then
    CURRENT_STALE_TASK=0
    fi
    }

    ##
    # Set the task definition revision/version and the desired task number of instances that the service should run.
    #
    # @params string $1 Version/revision the task definition (full ARN)
    # @params int $2 The desired number of task instances to be run.
    # @reads string $ECS_CLUSTER The name of the ECS cluster
    # @reads string $ECS_SERVICE The name of the ECS service
    # @reads string $AWS_ACCESS_KEY_ID AWS Access key
    # @reads string $AWS_DEFAULT_REGION AWS Default Region
    # @reads string $AWS_SECRET_ACCESS_KEY AWS Secret access key
    ##
    function update_ecs_service() {
    output=$(aws ecs update-service --cluster $ECS_CLUSTER \
    --service $ECS_SERVICE \
    --task-definition $1 \
    --desired-count $2)

    if [[ $(echo $output | $JQ '.service.taskDefinition') != $1 ]] || [[ $(echo $output | $JQ '.service.desiredCount') != $2 ]]; then
    echo -e "\n$(date "+%Y-%m-%d %H:%M:%S") Error, in setting service"
    exit 2
    fi
    }

    ##
    # Push/Update the task definition to ECS and set the newly created task revision full arn ($CURRENT_TASK_REVISION)
    #
    # @params string $1 Task Definition (JSON)
    # @reads string $ECS_FAMILY The task family (aka name)
    # @reads string $AWS_ACCESS_KEY_ID AWS Access key
    # @reads string $AWS_DEFAULT_REGION AWS Default Region
    # @reads string $AWS_SECRET_ACCESS_KEY AWS Secret access key
    # @sets string $CURRENT_TASK_REVISION The task revision/version (Full ARN)
    ##
    function update_ecs_task_def() {
    if CURRENT_TASK_REVISION=$(aws ecs register-task-definition --container-definitions "$1" \
    --family $ECS_FAMILY \
    | $JQ '.taskDefinition.taskDefinitionArn'); then
    echo -e "\n$(date "+%Y-%m-%d %H:%M:%S") Successfully register task definition :\n\tfamily : $ECS_FAMILY\n\tRevision : $CURRENT_TASK_REVISION\n"
    return 0
    fi

    echo -e "\n$(date "+%Y-%m-%d %H:%M:%S") Failed to register task definition :\n\tfamily : $ECS_FAMILY"
    exit 1
    }

    ##
    # Wait until the service has reached The desired number of running task instances
    #
    # @uses get_ecs_status()
    ##
    function wait_ecs_nb_task() {
    for attempt in {1..120}; do
    get_ecs_status
    if [ $CURRENT_RUNNING_TASK -ne $CURRENT_DESIRED_COUNT ]; then
    sleep 1
    else
    return 0
    fi
    done

    echo -e "\n\n$(date "+%Y-%m-%d %H:%M:%S") Waiting for running count to reach $CURRENT_DESIRED_COUNT took to long. Current running task : $CURRENT_RUNNING_TASK\n\n"
    exit 3
    }

    ##
    # Wait for all stale task to disappear.
    #
    # Stale task = Task of not the current version/revision
    #
    # @param string $1 The current task revision
    # @uses get_ecs_status()
    ##
    function wait_ecs_no_stale_task() {
    for attempt in {1..240}; do
    get_ecs_status;
    echo "$(date "+%Y-%m-%d %H:%M:%S") Running : $CURRENT_RUNNING_TASK, Desired : $CURRENT_DESIRED_COUNT, Stale : $CURRENT_STALE_TASK"

    if [[ $CURRENT_STALE_TASK>0 ]]; then
    sleep 2
    else
    return 0
    fi
    done

    echo "\n\nService update took too long.\n\n"
    exit 4
    }

    main "$@"
    exit 0