Created
February 27, 2015 20:21
-
-
Save slb350/12c9ce670ded1c9c25be to your computer and use it in GitHub Desktop.
EC2 - Polls w/Boto
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| import sys | |
| import os | |
| import argparse | |
| import re | |
| from time import time | |
| import boto | |
| from boto import ec2 | |
| from boto import rds | |
| from boto import route53 | |
| import ConfigParser | |
| from collections import defaultdict | |
| try: | |
| import json | |
| except ImportError: | |
| import simplejson as json | |
| class Ec2Inventory(object): | |
| def _empty_inventory(self): | |
| return {"_meta" : {"hostvars" : {}}} | |
| def __init__(self): | |
| self.inventory = self._empty_inventory() | |
| self.index = {} | |
| self.read_settings() | |
| self.parse_cli_args() | |
| if self.args.refresh_cache: | |
| self.do_api_calls_update_cache() | |
| elif not self.is_cache_valid(): | |
| self.do_api_calls_update_cache() | |
| if self.args.host: | |
| data_to_print = self.get_host_info() | |
| elif self.args.list: | |
| if self.inventory == self._empty_inventory(): | |
| data_to_print = self.get_inventory_from_cache() | |
| else: | |
| data_to_print = self.json_format_dict(self.inventory, True) | |
| print data_to_print | |
| def is_cache_valid(self): | |
| if os.path.isfile(self.cache_path_cache): | |
| mod_time = os.path.getmtime(self.cache_path_cache) | |
| current_time = time() | |
| if (mod_time + self.cache_max_age) > current_time: | |
| if os.path.isfile(self.cache_path_index): | |
| return True | |
| return False | |
| def read_settings(self): | |
| config = ConfigParser.SafeConfigParser() | |
| ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') | |
| ec2_ini_path = os.environ.get('EC2_INI_PATH', ec2_default_ini_path) | |
| config.read(ec2_ini_path) | |
| self.eucalyptus_host = None | |
| self.eucalyptus = False | |
| if config.has_option('ec2', 'eucalyptus'): | |
| self.eucalyptus = config.getboolean('ec2', 'eucalyptus') | |
| if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): | |
| self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') | |
| self.regions = [] | |
| configRegions = config.get('ec2', 'regions') | |
| configRegions_exclude = config.get('ec2', 'regions_exclude') | |
| if (configRegions == 'all'): | |
| if self.eucalyptus_host: | |
| self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) | |
| else: | |
| for regionInfo in ec2.regions(): | |
| if regionInfo.name not in configRegions_exclude: | |
| self.regions.append(regionInfo.name) | |
| else: | |
| self.regions = configRegions.split(",") | |
| self.destination_variable = config.get('ec2', 'destination_variable') | |
| self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') | |
| self.route53_enabled = config.getboolean('ec2', 'route53') | |
| self.route53_excluded_zones = [] | |
| if config.has_option('ec2', 'route53_excluded_zones'): | |
| self.route53_excluded_zones.extend( | |
| config.get('ec2', 'route53_excluded_zones', '').split(',')) | |
| self.rds_enabled = True | |
| if config.has_option('ec2', 'rds'): | |
| self.rds_enabled = config.getboolean('ec2', 'rds') | |
| if config.has_option('ec2', 'all_instances'): | |
| self.all_instances = config.getboolean('ec2', 'all_instances') | |
| else: | |
| self.all_instances = False | |
| if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled: | |
| self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') | |
| else: | |
| self.all_rds_instances = False | |
| cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) | |
| if not os.path.exists(cache_dir): | |
| os.makedirs(cache_dir) | |
| self.cache_path_cache = cache_dir + "/ansible-ec2.cache" | |
| self.cache_path_index = cache_dir + "/ansible-ec2.index" | |
| self.cache_max_age = config.getint('ec2', 'cache_max_age') | |
| if config.has_option('ec2', 'nested_groups'): | |
| self.nested_groups = config.getboolean('ec2', 'nested_groups') | |
| else: | |
| self.nested_groups = False | |
| try: | |
| pattern_include = config.get('ec2', 'pattern_include') | |
| if pattern_include and len(pattern_include) > 0: | |
| self.pattern_include = re.compile(pattern_include) | |
| else: | |
| self.pattern_include = None | |
| except ConfigParser.NoOptionError, e: | |
| self.pattern_include = None | |
| try: | |
| pattern_exclude = config.get('ec2', 'pattern_exclude'); | |
| if pattern_exclude and len(pattern_exclude) > 0: | |
| self.pattern_exclude = re.compile(pattern_exclude) | |
| else: | |
| self.pattern_exclude = None | |
| except ConfigParser.NoOptionError, e: | |
| self.pattern_exclude = None | |
| self.ec2_instance_filters = defaultdict(list) | |
| if config.has_option('ec2', 'instance_filters'): | |
| for x in config.get('ec2', 'instance_filters', '').split(','): | |
| filter_key, filter_value = x.split('=') | |
| self.ec2_instance_filters[filter_key].append(filter_value) | |
| def parse_cli_args(self): | |
| parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') | |
| parser.add_argument('--list', action='store_true', default=True, | |
| help='List instances (default: True)') | |
| parser.add_argument('--host', action='store', | |
| help='Get all the variables about a specific instance') | |
| parser.add_argument('--refresh-cache', action='store_true', default=False, | |
| help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') | |
| self.args = parser.parse_args() | |
| def do_api_calls_update_cache(self): | |
| if self.route53_enabled: | |
| self.get_route53_records() | |
| for region in self.regions: | |
| self.get_instances_by_region(region) | |
| if self.rds_enabled: | |
| self.get_rds_instances_by_region(region) | |
| self.write_to_cache(self.inventory, self.cache_path_cache) | |
| self.write_to_cache(self.index, self.cache_path_index) | |
| def get_instances_by_region(self, region): | |
| try: | |
| if self.eucalyptus: | |
| conn = boto.connect_euca(host=self.eucalyptus_host) | |
| conn.APIVersion = '2010-08-31' | |
| else: | |
| conn = ec2.connect_to_region(region) | |
| if conn is None: | |
| print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) | |
| sys.exit(1) | |
| reservations = [] | |
| if self.ec2_instance_filters: | |
| for filter_key, filter_values in self.ec2_instance_filters.iteritems(): | |
| reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) | |
| else: | |
| reservations = conn.get_all_instances() | |
| for reservation in reservations: | |
| for instance in reservation.instances: | |
| self.add_instance(instance, region) | |
| except boto.exception.BotoServerError, e: | |
| if not self.eucalyptus: | |
| print "Looks like AWS is down again:" | |
| print e | |
| sys.exit(1) | |
| def get_rds_instances_by_region(self, region): | |
| try: | |
| conn = rds.connect_to_region(region) | |
| if conn: | |
| instances = conn.get_all_dbinstances() | |
| for instance in instances: | |
| self.add_rds_instance(instance, region) | |
| except boto.exception.BotoServerError, e: | |
| if not e.reason == "Forbidden": | |
| print "Looks like AWS RDS is down: " | |
| print e | |
| sys.exit(1) | |
| def get_instance(self, region, instance_id): | |
| ''' Gets details about a specific instance ''' | |
| if self.eucalyptus: | |
| conn = boto.connect_euca(self.eucalyptus_host) | |
| conn.APIVersion = '2010-08-31' | |
| else: | |
| conn = ec2.connect_to_region(region) | |
| if conn is None: | |
| print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) | |
| sys.exit(1) | |
| reservations = conn.get_all_instances([instance_id]) | |
| for reservation in reservations: | |
| for instance in reservation.instances: | |
| return instance | |
| def add_instance(self, instance, region): | |
| ''' Adds an instance to the inventory and index, as long as it is | |
| addressable ''' | |
| if not self.all_instances and instance.state != 'running': | |
| return | |
| if instance.subnet_id: | |
| dest = getattr(instance, self.vpc_destination_variable) | |
| else: | |
| dest = getattr(instance, self.destination_variable) | |
| if not dest: | |
| return | |
| if self.pattern_include and not self.pattern_include.match(dest): | |
| return | |
| if self.pattern_exclude and self.pattern_exclude.match(dest): | |
| return | |
| self.index[dest] = [region, instance.id] | |
| self.inventory[instance.id] = [dest] | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'instances', instance.id) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'regions', region) | |
| else: | |
| self.push(self.inventory, region, dest) | |
| self.push(self.inventory, instance.placement, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, region, instance.placement) | |
| type_name = self.to_safe('type_' + instance.instance_type) | |
| self.push(self.inventory, type_name, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'types', type_name) | |
| if instance.key_name: | |
| key_name = self.to_safe('key_' + instance.key_name) | |
| self.push(self.inventory, key_name, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'keys', key_name) | |
| if instance.vpc_id: | |
| self.push(self.inventory, self.to_safe('vpc_id_' + instance.vpc_id), dest) | |
| try: | |
| for group in instance.groups: | |
| key = self.to_safe("security_group_" + group.name) | |
| self.push(self.inventory, key, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'security_groups', key) | |
| except AttributeError: | |
| print 'Package boto seems a bit older.' | |
| print 'Please upgrade boto >= 2.3.0.' | |
| sys.exit(1) | |
| for k, v in instance.tags.iteritems(): | |
| key = self.to_safe("tag_" + k + "=" + v) | |
| self.push(self.inventory, key, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) | |
| self.push_group(self.inventory, self.to_safe("tag_" + k), key) | |
| if self.route53_enabled: | |
| route53_names = self.get_instance_route53_names(instance) | |
| for name in route53_names: | |
| self.push(self.inventory, name, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'route53', name) | |
| if len(instance.tags) == 0: | |
| self.push(self.inventory, 'tag_none', dest) | |
| self.push(self.inventory, 'ec2', dest) | |
| self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) | |
| def add_rds_instance(self, instance, region): | |
| if not self.all_rds_instances and instance.status != 'available': | |
| return | |
| dest = instance.endpoint[0] | |
| if not dest: | |
| return | |
| self.index[dest] = [region, instance.id] | |
| self.inventory[instance.id] = [dest] | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'instances', instance.id) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'regions', region) | |
| else: | |
| self.push(self.inventory, region, dest) | |
| self.push(self.inventory, instance.availability_zone, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, region, instance.availability_zone) | |
| type_name = self.to_safe('type_' + instance.instance_class) | |
| self.push(self.inventory, type_name, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'types', type_name) | |
| try: | |
| if instance.security_group: | |
| key = self.to_safe("security_group_" + instance.security_group.name) | |
| self.push(self.inventory, key, dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'security_groups', key) | |
| except AttributeError: | |
| print 'Package boto seems a bit older.' | |
| print 'Please upgrade boto >= 2.3.0.' | |
| sys.exit(1) | |
| self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) | |
| self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest) | |
| if self.nested_groups: | |
| self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) | |
| self.push(self.inventory, 'rds', dest) | |
| self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) | |
| def get_route53_records(self): | |
| r53_conn = route53.Route53Connection() | |
| all_zones = r53_conn.get_zones() | |
| route53_zones = [ zone for zone in all_zones if zone.name[:-1] | |
| not in self.route53_excluded_zones ] | |
| self.route53_records = {} | |
| for zone in route53_zones: | |
| rrsets = r53_conn.get_all_rrsets(zone.id) | |
| for record_set in rrsets: | |
| record_name = record_set.name | |
| if record_name.endswith('.'): | |
| record_name = record_name[:-1] | |
| for resource in record_set.resource_records: | |
| self.route53_records.setdefault(resource, set()) | |
| self.route53_records[resource].add(record_name) | |
| def get_instance_route53_names(self, instance): | |
| instance_attributes = [ 'public_dns_name', 'private_dns_name', | |
| 'ip_address', 'private_ip_address' ] | |
| name_list = set() | |
| for attrib in instance_attributes: | |
| try: | |
| value = getattr(instance, attrib) | |
| except AttributeError: | |
| continue | |
| if value in self.route53_records: | |
| name_list.update(self.route53_records[value]) | |
| return list(name_list) | |
| def get_host_info_dict_from_instance(self, instance): | |
| instance_vars = {} | |
| for key in vars(instance): | |
| value = getattr(instance, key) | |
| key = self.to_safe('ec2_' + key) | |
| if key == 'ec2__state': | |
| instance_vars['ec2_state'] = instance.state or '' | |
| instance_vars['ec2_state_code'] = instance.state_code | |
| elif key == 'ec2__previous_state': | |
| instance_vars['ec2_previous_state'] = instance.previous_state or '' | |
| instance_vars['ec2_previous_state_code'] = instance.previous_state_code | |
| elif type(value) in [int, bool]: | |
| instance_vars[key] = value | |
| elif type(value) in [str, unicode]: | |
| instance_vars[key] = value.strip() | |
| elif type(value) == type(None): | |
| instance_vars[key] = '' | |
| elif key == 'ec2_region': | |
| instance_vars[key] = value.name | |
| elif key == 'ec2__placement': | |
| instance_vars['ec2_placement'] = value.zone | |
| elif key == 'ec2_tags': | |
| for k, v in value.iteritems(): | |
| key = self.to_safe('ec2_tag_' + k) | |
| instance_vars[key] = v | |
| elif key == 'ec2_groups': | |
| group_ids = [] | |
| group_names = [] | |
| for group in value: | |
| group_ids.append(group.id) | |
| group_names.append(group.name) | |
| instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) | |
| instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) | |
| else: | |
| pass | |
| return instance_vars | |
| def get_host_info(self): | |
| if len(self.index) == 0: | |
| self.load_index_from_cache() | |
| if not self.args.host in self.index: | |
| self.do_api_calls_update_cache() | |
| if not self.args.host in self.index: | |
| return self.json_format_dict({}, True) | |
| (region, instance_id) = self.index[self.args.host] | |
| instance = self.get_instance(region, instance_id) | |
| return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) | |
| def push(self, my_dict, key, element): | |
| group_info = my_dict.setdefault(key, []) | |
| if isinstance(group_info, dict): | |
| host_list = group_info.setdefault('hosts', []) | |
| host_list.append(element) | |
| else: | |
| group_info.append(element) | |
| def push_group(self, my_dict, key, element): | |
| parent_group = my_dict.setdefault(key, {}) | |
| if not isinstance(parent_group, dict): | |
| parent_group = my_dict[key] = {'hosts': parent_group} | |
| child_groups = parent_group.setdefault('children', []) | |
| if element not in child_groups: | |
| child_groups.append(element) | |
| def get_inventory_from_cache(self): | |
| cache = open(self.cache_path_cache, 'r') | |
| json_inventory = cache.read() | |
| return json_inventory | |
| def load_index_from_cache(self): | |
| cache = open(self.cache_path_index, 'r') | |
| json_index = cache.read() | |
| self.index = json.loads(json_index) | |
| def write_to_cache(self, data, filename): | |
| json_data = self.json_format_dict(data, True) | |
| cache = open(filename, 'w') | |
| cache.write(json_data) | |
| cache.close() | |
| def to_safe(self, word): | |
| return re.sub("[^A-Za-z0-9\-]", "_", word) | |
| def json_format_dict(self, data, pretty=False): | |
| if pretty: | |
| return json.dumps(data, sort_keys=True, indent=2) | |
| else: | |
| return json.dumps(data) | |
| Ec2Inventory() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment