Skip to content

Instantly share code, notes, and snippets.

@shoppersstop-support

shoppersstop-support/environment_provisioning.py

Created July 24, 2018 05:40
Show Gist options
  • Save shoppersstop-support/acca417fed736d5e93cd96bffcfb766f to your computer and use it in GitHub Desktop.
Save shoppersstop-support/acca417fed736d5e93cd96bffcfb766f to your computer and use it in GitHub Desktop.
Download ZIP
Raw
environment_provisioning.py
import boto3
import sys
from datetime import datetime, timedelta, date
import json
# import jenkins
import base64
import spotinst_sdk
import time
import re
ACCESS_KEY = '******'
SECRET_KEY = '******'
REGION = 'ap-south-1'
AUTH_TOKE = '******'
ACCOUNT_ID = 'act-918ef916'
INSTANCE_KEY_PAIR = 'ss-mum-uat'
client = boto3.client('ec2', region_name='ap-south-1', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY)
target_group_client = boto3.client('elbv2', region_name='ap-south-1', aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY)
INSTANCE_NAME = str(sys.argv[1])
DESTINATION_ENVIRONMENT = "Test"
FQDE = "Test-8"
TODAY = datetime.now().strftime('%Y-%m-%d')
TIME = datetime.now().strftime('%d-%m-%Y-%H-%M')
instance_name = INSTANCE_NAME.split(",")
FQDE_2 = re.sub('-', '', FQDE)
LOWER_FQDN = FQDE_2.lower()
UPPER_FQDE = FQDE_2.upper()
UPPER_FQDE = UPPER_FQDE
LOWER_FQDN = LOWER_FQDN
RDS_INSTANCE_IDENTIFIER = LOWER_FQDN
RDS_SNAPSHOT_IDENTIFIER = "oracle-prod2018-06-18-19-30-06"
NEW_RDS_SNAPSHOT_IDENTIFIER = "abcd"+TIME
SNAPSHOT = "Existing"
def get_instance_id():
global subnet_dic
try:
for iname in instance_name:
response = client.describe_instances(Filters=[{'Name': 'tag:' + 'Name', 'Values': [iname]}])
for reservation in response["Reservations"]:
for i in reservation["Instances"]:
instance_id = i["InstanceId"]
print iname + " Instance ID is: " + instance_id
tags = get_instance_tag(iname, instance_id)
# craete_image(instance_id,iname,tags)
get_security_group(security_group_tag)
subnet_dic = get_subnet_id(subnet_tag)
# create_target_group()
# create_rule()
ami_ids = "ami-6198b30e"
# defining_spotinst_requirements(ami_ids, securityGroupId, SSH_SECURITY_GROUP, subnet_dic)
except Exception as e:
print(e)
def get_instance_tag(iname, instance_id):
global security_group_tag, subnet_tag, source_instance_layer
response = client.describe_instances(InstanceIds=[instance_id])
for reservation in response["Reservations"]:
for i in reservation["Instances"]:
tags = i["Tags"]
for tag in i["Tags"]:
if tag['Key'] == 'Layer':
source_instance_layer = tag['Value']
print "Source instance Layer : " + source_instance_layer
if tag['Key'] == 'Environment':
source_instance_environment = tag['Value']
print "Source instance Environment : " + source_instance_environment
security_group_tag = {'Environment': DESTINATION_ENVIRONMENT, 'Layer': source_instance_layer}
subnet_tag = {'Environment': DESTINATION_ENVIRONMENT, 'Layer': source_instance_layer}
return tags
def craete_image(instance_id, iname, tags):
global ami_ids
response = client.create_image(InstanceId=instance_id, Name=iname + '-' + TIME, NoReboot=True)
ami_ids = response['ImageId']
print "AMI-ID is: " + ami_ids
apply_ami_tags(tags, ami_ids)
check_ami_avilability(ami_ids)
print iname + " AMI Id is: " + ami_ids
def apply_ami_tags(instance_tags, ami_ids):
create_tags = client.create_tags(Resources=[ami_ids], Tags=[{'Key': "DeleteOn", 'Value': TODAY}])
apply_instance_tag_to_ami = client.create_tags(Resources=[ami_ids], Tags=instance_tags)
def check_ami_avilability(ami_ids):
try:
available = 0
while available == 0:
print "Not created yet.. Gonna sleep for 10 seconds"
time.sleep(10)
image = client.describe_images(ImageIds=[ami_ids])
if image['Images'][0]['State'] == 'available':
available = 1
if available == 1:
print "Image is now available for use."
return True
except Exception, e:
print e
def get_security_group(security_group_tag):
global securityGroupId, SSH_SECURITY_GROUP
filters = list();
for key, value in security_group_tag.iteritems():
filters.extend([{'Name': "tag-key", 'Values': [key]}, {'Name': "tag-value", 'Values': [value]}])
try:
response = client.describe_security_groups(Filters=filters)
for sg in response["SecurityGroups"]:
securityGroupId = sg["GroupId"]
print securityGroupId
if DESTINATION_ENVIRONMENT == "Test":
SSH_SECURITY_GROUP = "sg-d9f17eb1"
IAM_ROLE = "`"
elif DESTINATION_ENVIRONMENT == "UAT":
SSH_SECURITY_GROUP = "sg-b975fcd1"
else:
exit(0)
print "SSH SG Name is: " + SSH_SECURITY_GROUP
except Exception as e:
print(e)
def get_subnet_id(subnet_tag):
global subnetid, az
filters = list();
subnet_dic = {}
for key, value in subnet_tag.iteritems():
filters.extend([{'Name': "tag-key", 'Values': [key]}, {'Name': "tag-value", 'Values': [value]}])
try:
response = client.describe_subnets(Filters=filters)
for subnet in response["Subnets"]:
# print subnet
subnetid = subnet["SubnetId"]
az = subnet["AvailabilityZone"]
print subnetid
print az
subnet_dic.setdefault(az, []).append(subnetid)
return subnet_dic
except Exception as e:
print(e)
def create_target_group():
small_source_instance_layer = source_instance_layer.lower()
small_source_instance_layer = small_source_instance_layer
global http_target_group_arn, https_target_group_arn
http_target_group_response = target_group_client.create_target_group(
Name=LOWER_FQDN + "-" + small_source_instance_layer + '-' + "9001", Protocol='HTTP', Port=9001,
VpcId='vpc-2c80b545', HealthCheckProtocol='HTTP',
HealthCheckPort='traffic-port', HealthCheckPath='/', HealthCheckIntervalSeconds=10,
HealthCheckTimeoutSeconds=5, HealthyThresholdCount=5, UnhealthyThresholdCount=2,
Matcher={'HttpCode': '302'}, TargetType='instance')
https_target_group_response = target_group_client.create_target_group(
Name=LOWER_FQDN + "-" + small_source_instance_layer + '-' + "9002", Protocol='HTTPS', Port=9002,
VpcId='vpc-2c80b545', HealthCheckProtocol='HTTPS',
HealthCheckPort='traffic-port', HealthCheckPath='/', HealthCheckIntervalSeconds=10,
HealthCheckTimeoutSeconds=5, HealthyThresholdCount=5, UnhealthyThresholdCount=2,
Matcher={'HttpCode': '200'}, TargetType='instance')
for response in http_target_group_response["TargetGroups"]:
http_target_group_arn = response["TargetGroupArn"]
print http_target_group_arn
for response in https_target_group_response["TargetGroups"]:
https_target_group_arn = response["TargetGroupArn"]
print https_target_group_arn
def create_rule():
if source_instance_layer == "App":
host_header_value = LOWER_FQDN + ".shopper-stop.in"
elif source_instance_layer == "Admin":
host_header_value = "admin" + LOWER_FQDN + ".shopper-stop.in"
elif source_instance_layer == "Omm":
host_header_value = "omm" + LOWER_FQDN + ".shopper-stop.in"
else:
exit(0)
response = target_group_client.create_rule(
ListenerArn='arn:aws:elasticloadbalancing:ap-south-1:099729443697:listener/app/test-alb/a45e5bf420c2a71f/acbd2bfb4a9b6951',
Conditions=[{'Field': 'host-header', 'Values': [host_header_value]}], Priority=1,
Actions=[{'Type': 'forward', 'TargetGroupArn': http_target_group_arn}])
response = target_group_client.create_rule(
ListenerArn='arn:aws:elasticloadbalancing:ap-south-1:099729443697:listener/app/test-alb/a45e5bf420c2a71f/78e5e10f696021cf',
Conditions=[{'Field': 'host-header', 'Values': [host_header_value]}], Priority=2000,
Actions=[{'Type': 'forward', 'TargetGroupArn': https_target_group_arn}])
def defining_spotinst_requirements(ami_ids, securityGroupId, SSH_SECURITY_GROUP, subnet_dic):
small_source_instance_layer = source_instance_layer.lower()
small_source_instance_layer = small_source_instance_layer
ELASTIC_GROUP_NAME = UPPER_FQDE + "-" + small_source_instance_layer.upper()
client = spotinst_sdk.SpotinstClient(auth_token=AUTH_TOKE, account_id=ACCOUNT_ID)
strategy = spotinst_sdk.aws_elastigroup.Strategy(risk=100, utilize_reserved_instances=False, fallback_to_od=True,
availability_vs_cost="balanced")
capacity = spotinst_sdk.aws_elastigroup.Capacity(minimum=0, maximum=0, target=0, unit="instance")
iam_roll = spotinst_sdk.aws_elastigroup.IamRole(name='test')
securityGroupId = securityGroupId
SSH_SECURITY_GROUP = SSH_SECURITY_GROUP
securityGroupIds = [securityGroupId, SSH_SECURITY_GROUP]
# Initialize group tags
tag1 = spotinst_sdk.aws_elastigroup.Tag(tag_key="Environment", tag_value=FQDE)
tag2 = spotinst_sdk.aws_elastigroup.Tag(tag_key="Layer", tag_value=source_instance_layer)
tag3 = spotinst_sdk.aws_elastigroup.Tag(tag_key="Name", tag_value=LOWER_FQDN + "-" + small_source_instance_layer)
tag4 = spotinst_sdk.aws_elastigroup.Tag(tag_key="Deploy", tag_value="True")
tag5 = spotinst_sdk.aws_elastigroup.Tag(tag_key="Platform", tag_value="Hybris")
if source_instance_layer == "App":
tags = [tag1, tag2, tag3, tag4, tag5]
elif source_instance_layer == "Search":
tags = [tag1, tag2, tag3]
else:
exit(0)
a = 'home/minjar/scripts/sensu-client-remove.sh\r/bin/bash -x /home/minjar/scripts/shutdown.sh\r'
b = base64.b64encode(a.encode('utf8'))
scripts = b
launchSpec = spotinst_sdk.aws_elastigroup.LaunchSpecification(image_id=ami_ids, tags=tags, iam_role=iam_roll, shutdown_script=scripts,
key_pair=INSTANCE_KEY_PAIR,
security_group_ids=securityGroupIds, monitoring=False)
az_list = []
for az in subnet_dic.keys():
subnetid = subnet_dic[az]
az1 = spotinst_sdk.aws_elastigroup.AvailabilityZone(name=az, subnet_ids=subnetid)
az_list.append(az1)
print az_list
instance_types = spotinst_sdk.aws_elastigroup.InstanceTypes(ondemand="t2.micro", spot=["t2.micro","t2.small"])
compute = spotinst_sdk.aws_elastigroup.Compute(product="Linux/UNIX", instance_types=instance_types,
availability_zones=az_list, launch_specification=launchSpec)
group = spotinst_sdk.aws_elastigroup.Elastigroup(name=ELASTIC_GROUP_NAME,
description="Created by the Python SDK",
capacity=capacity, strategy=strategy, compute=compute)
print "-------Create Group-------"
# Create elastigroup and retrieve group id
group = client.create_elastigroup(group)
group_id = group['id']
# Retrieve some internal property
utilize_reserved = group['strategy']['utilize_reserved_instances']
print group_id
print("-------Get Group-------")
# Retrieve a single elastigroup
group = client.get_elastigroup(group_id=group_id)
print("-------Update Group-------")
capacity_update = spotinst_sdk.aws_elastigroup.Capacity(minimum=0, maximum=0, target=0)
strategy_update = spotinst_sdk.aws_elastigroup.Strategy(risk=100)
ami_auto_backup = spotinst_sdk.aws_elastigroup.ScheduledTask(frequency='hourly',task_type='backup_ami')
schdeduling_2 = spotinst_sdk.aws_elastigroup.ScheduledTask(task_type='scale' , is_enabled=False)
scale_up = spotinst_sdk.aws_elastigroup.ScheduledTask(cron_expression='30 02 * * 1-5',task_type='scale',
scale_target_capacity=1, scale_min_capacity=1, scale_max_capacity=1)
scale_down = spotinst_sdk.aws_elastigroup.ScheduledTask(cron_expression='30 17 * * 1-5', task_type='scale',
scale_target_capacity=0, scale_min_capacity=0, scale_max_capacity=1)
schdeduling = spotinst_sdk.aws_elastigroup.Scheduling(tasks=[ ami_auto_backup, scale_up, scale_down], )
group_update = spotinst_sdk.aws_elastigroup.Elastigroup(capacity=capacity_update, strategy=strategy_update, scheduling=schdeduling,)
group_after_update = client.update_elastigroup(group_update=group_update, group_id=group_id)
print(group_after_update)
print("-------Update Only AMI-------")
update_lspec = spotinst_sdk.aws_elastigroup.LaunchSpecification(image_id=ami_ids)
update_compute = spotinst_sdk.aws_elastigroup.Compute(launch_specification=update_lspec)
group_update = spotinst_sdk.aws_elastigroup.Elastigroup(compute=update_compute)
group_after_update = client.update_elastigroup(group_update=group_update, group_id=group_id)
print(group_after_update)
print("-------Roll Group-------")
group_roll = spotinst_sdk.aws_elastigroup.Roll(batch_size_percentage=1, grace_period=600)
roll_response = client.roll_group(group_id=group_id, group_roll=group_roll)
print(roll_response)
def rds_creation():
client = boto3.client('rds', region_name='ap-south-1', aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY)
if SNAPSHOT == "NEW":
client = client.restore_db_instance_from_db_snapshot(DBInstanceIdentifier=RDS_INSTANCE_IDENTIFIER,
DBSnapshotIdentifier=NEW_RDS_SNAPSHOT_IDENTIFIER,
DBInstanceClass='db.t2.micro', Port=1521,
DBSubnetGroupName="test-db-subnet-group",
MultiAZ=False,
PubliclyAccessible=False,
AutoMinorVersionUpgrade=True, DBName="ORCL",
OptionGroupName="test-env-og",
StorageType="gp2", CopyTagsToSnapshot=False,
Tags=[{'Key': 'Environment', 'Value': FQDE},
{'Key': "Layer", 'Value': "DB"}])
time.sleep(180)
try:
available = 0
while available == 0:
print "RDS instance not created yet.. Gonna sleep for 10 seconds"
time.sleep(10)
rds_response = client.describe_db_instances(DBInstanceIdentifier=RDS_INSTANCE_IDENTIFIER)
if rds_response['DBInstances'][0]['DBInstanceStatus'] == 'available':
available = 1
if available == 1:
print "RDS is now available for use."
return True
except Exception, e:
print e
elif SNAPSHOT == "Existing":
client = client.restore_db_instance_from_db_snapshot(DBInstanceIdentifier=RDS_INSTANCE_IDENTIFIER,
DBSnapshotIdentifier=RDS_SNAPSHOT_IDENTIFIER,
DBInstanceClass='db.t2.micro', Port=1521,
DBSubnetGroupName="test-db-subnet-group",
MultiAZ=False,
PubliclyAccessible=False,
AutoMinorVersionUpgrade=False, DBName="ORCL",
OptionGroupName="test-env-og",
StorageType="gp2", CopyTagsToSnapshot=False,
Tags=[{'Key': 'Environment', 'Value': "Test-7"},
{'Key': "Layer", 'Value': "DB"}])
print "RDS instance is in creating staage.. Gonna sleep for 3 min..."
time.sleep(180)
try:
available = 0
while available == 0:
print "RDS instance not created yet.. Gonna sleep for 10 seconds"
time.sleep(10)
rds_response = client.describe_db_instances(Filters=[{'Name': 'Environment','Values': ['Test-7']}])
if rds_response['DBInstances'][0]['DBInstanceStatus'] == 'available':
available = 1
if available == 1:
print "RDS is now available for use."
return True
except Exception, e:
print e
else:
exit(0)
get_instance_id()
rds_creation()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment