# -*- coding: utf-8 -*- #Copyright (C) 2013 Seán Hayes import my_project.settings as dj_settings from fabric.api import local, run, sudo, env, prompt, settings, cd, parallel, execute from fabric.contrib.files import exists from fabric.decorators import hosts, roles, runs_once import json import logging import os logging.getLogger('').setLevel(logging.INFO) logger = logging.getLogger(__name__) env.forward_agent = True env.user = 'username' role_list = ( 'web', 'cache', 'db', 'celery', ) host_dict = { '100.100.100.100': ('web', 'cache',), '100.100.100.101': ('db', 'celery',), } env.hosts = host_dict.keys() for r in role_list: env.roledefs[r] = [] for k_host, v_roles in host_dict.items(): for v_role in v_roles: env.roledefs[v_role].append(k_host) env.code_dir = '/srv/' env.package_name = dj_settings.PACKAGE_MODULE env.project_dir = '%s%s/' % (env.code_dir, dj_settings.PROJECT_NAME,) env.package_dir = '%s%s/' % (env.project_dir, dj_settings.PACKAGE_MODULE,) env.project_git_uri = 'git@github.com:some-user/my-project.git' env.config_dir = '%sconfig/generated/' % env.project_dir env.log_dir = '%slogs/' % env.project_dir env.pip_dir = '%spip/' % env.code_dir env.celery_script_dir = '%scelery/init.d/' % env.config_dir main_dirs = [ env.pip_dir, ] project_dirs = [ env.log_dir, ] apt_packages = [ 'debconf-utils', 'git', 'mercurial', 'subversion', 'ntp', 'gdebi-core', 'graphviz', 'graphviz-dev', 'libmemcached-tools', 'memcached', 'nginx', 'pkg-config', 'postfix', 'python-pip', 'python-virtualenv', 'python-all-dev', 'postgresql', #'rabbitmq-server', #TODO: try to get as many of these as possible in requirements.txt #'python-django-doc', #some require 1/4 GB of dependencies to build the PIP version, which is unacceptable for this kind of application 'python-imaging', 'python-psycopg2', #'python-exactimage', #'python-crypto', ] #Run the following to make binary eggs when setuptools isn't used #python -c "import setuptools; execfile('setup.py')" bdist_egg # tasks def create_user(): "Create admin user on fresh cloud instance." username = prompt('Enter username to create: ', default=env.user) with settings(user='root'): run('useradd --groups sudo,www-data -d /home/%s -m %s' % (username, username)) run('passwd %s' % username) def switch_to_bash(): "switch from dash (the Ubuntu default) to bash" with cd('/bin'): #has to be one command since each call to sudo() is a different session, #and you can't login if sh isn't set sudo('rm sh; ln -s bash sh') @roles('db') @runs_once def setup_pgsql(): "Sets up PostgreSQL user and databases." name = prompt('Enter PostgreSQL role/db to create: ', default=env.project_name) sudo('createuser -s -P %s' % name, user='postgres') sudo('createdb -O %s %s' % (name, name), user='postgres') def mkdirs(dirs): "Sets up the directories we need and sets the right permissions." for d in dirs: if not exists(d): sudo('mkdir %s' % d) sudo('chown %s:www-data %s' % (env.user, d)) sudo('chmod 775 %s' % d) def upgrade_ubuntu(): "Probably shouldn't run this through Fabric, but here's the commands for it anyway." sudo('apt-get install update-manager-core') #edit /etc/update-manager/release-upgrades, set Prompt=normal sudo('do-release-upgrade') @parallel def install_apt(): "Updates package list, upgrades all packages to latest available version, and installs Apt dependencies for this project." sudo('apt-get update') sudo('apt-get upgrade') sudo('apt-get install -f %s' % ' '.join(apt_packages)) @parallel def set_permissions(): sudo('chown :www-data %s' % (env.code_dir,)) sudo('chmod 775 %s' % env.code_dir) @parallel def install_pip(): "Installs the PIP requirements for this project." with cd(env.pip_dir): sudo('pip install -r %srequirements.txt' % env.project_dir) @parallel def install_project(): "Clones this project's Git repo if there's no copy on the target machine, else it pulls the latest version." if exists(env.project_dir): with cd(env.project_dir): run('git pull origin master') else: with cd(env.code_dir): run('git clone %s' % env.project_git_uri) sudo('chown -R %s:www-data %s' % (env.user, env.project_dir)) sudo('chmod 775 %s' % env.project_dir) mkdirs(project_dirs) @hosts('') def install(): "Runs the commands to create all necessary directories, install Apt and PIP dependencies, and install project files." execute(mkdirs, main_dirs) execute(install_apt) execute(set_permissions) execute(install_project) execute(install_pip) @roles('db') @runs_once def migrate(): with cd(env.package_dir): run('./manage.py syncdb --migrate') @roles('web') def collectstatic(): with cd(env.project_dir): run('./manage.py collectstatic -l') def refresh_config_files(): "Regenerates dynamic config files using django-config-gen." with cd(env.package_dir): run('./manage.py config_gen') def link_config_file(source, destination): with settings(warn_only=True): sudo('rm %s' % destination) sudo('ln -s %s %s' % (source, destination)) @roles('web') def config_nginx(): with settings(warn_only=True): sudo('rm /etc/nginx/sites-available/*') link_config_file(os.path.join(env.config_dir, 'nginx'), '/etc/nginx/sites-available/default') @roles('db') def config_postgresql(): link_config_file(os.path.join(env.config_dir, 'pg_hba.conf'), '/etc/postgresql/9.1/main/pg_hba.conf') @roles('cache') def config_memcached(): link_config_file(os.path.join(env.config_dir, 'memcached.conf'), '/etc/memcached.conf') @roles('celery') def config_celery(): "Links Celery's Debian init scripts to /etc/init.d/." init_list=run('ls %s' % env.celery_script_dir).split() for script in init_list: p = '/etc/init.d/%s' % script init_file = os.path.join(env.celery_script_dir, script) link_config_file(init_file, p) sudo('chmod +x %s' % init_file) link_config_file(os.path.join(env.config_dir, 'celery/celeryd_default'), '/etc/default/celeryd') def config_tzdata(): "Configures the time zone for the server." run('echo \'America/New_York\'| sudo tee /etc/timezone') sudo('dpkg-reconfigure -f noninteractive tzdata') @hosts('') def config(): "Runs the commands to generate config files using django-config-gen and symlinks the generated files to the normal config file locations for Apache, Nginx, Memcached, etc." execute(refresh_config_files) execute(config_nginx) execute(config_memcached) execute(config_celery) execute(config_tzdata) @roles('web') def reload_nginx(): sudo('/etc/init.d/nginx reload') @roles('web') def reload_uwsgi(): sudo('kill -HUP `cat %suwsgi.pid`' % env.project_dir) @roles('celery') def restart_celery(): sudo('/etc/init.d/celeryd restart') sudo('/etc/init.d/celerybeat restart') sudo('/etc/init.d/celeryevcam restart') @roles('cache') def restart_memcached(): sudo('/etc/init.d/memcached restart') @hosts('') def reload_servers(): "Reloads Apache, Nginx, Rabbit MQ, and Celery where possible, otherwise it restarts them. Reloading config files is faster than restarting the processes." execute(reload_nginx) execute(reload_uwsgi) #sudo('/etc/init.d/rabbitmq-server reload') execute(restart_celery) execute(restart_memcached) #local development scripts def setup_git_shortcuts(): #https://git.wiki.kernel.org/index.php/Aliases shortcuts = { 'st': 'status', } for shortcut in shortcuts.items(): local('git config --global alias.%s %s' % shortcut) def set_django_colors(): local('export DJANGO_COLORS="%s"' % dj_settings.DJANGO_COLORS) def check_for_pdb(): "Easily check for instances of pdb.set_trace() in your code before committing." local('find . -name \'*.py\'|xargs grep \'pdb.set_trace\'') #TODO: handle exit code default_dump_file_template = '%s-dump.sql' def dump_db(): name = prompt('Enter PostgreSQL db to dump: ', default=env.project_name) local('sudo -u postgres pg_dump %s > %s' % (name, default_dump_file_template % name)) def load_db_dump(): name = prompt('Enter PostgreSQL db to load: ', default=env.project_name) local('sudo -u postgres psql %s < %s' % (name, default_dump_file_template % name))