Skip to content

Instantly share code, notes, and snippets.

@andrewschoen

andrewschoen/gist:9843936

Created March 28, 2014 22:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save andrewschoen/9843936 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save andrewschoen/9843936 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. andrewschoen created this gist Mar 28, 2014.
    80 changes: 80 additions & 0 deletions gistfile1.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,80 @@
    # NOTE: Needs it's own venv with latest psycopg2

    import os
    import psycopg2
    from psycopg2.extras import DictCursor
    import json
    import time
    import datetime

    SLEEP_SECONDS = 10

    DATABASE_DSNS = {
    'master': 'dbname=frank',
    }

    OUTPUT_DIR = '/Users/frank/work/revsys/clients/cmg/'

    HOST = 'logvipprd1.ddtc.cmgdigital.com'
    PORT = 5959

    HOSTMAP = {
    'master': 'pgldbaprd3',
    }


    class LogstashHandler(logging.handlers.DatagramHandler):

    def makePickle(self, record):
    msg = record.getMessage()
    data = json.loads(msg)
    data['@source_host'] = data['host']

    # Get the non @ fields
    data['@fields'] = [k for k in data if not k.startswith('@')]

    return json.dumps(data)


    logger = logging.getLogger(__name__)
    logger.addHandler(LogstashHandler(HOST, PORT))
    logger.setLevel(logging.DEBUG)


    def make_filename(name):
    time_string = datetime.datetime.now().strftime("%Y-%m-%d-hour-%H")
    return os.path.join(OUTPUT_DIR, '{0}-{1}.locks.log'.format(name, time_string))


    class DTEncoder(json.JSONEncoder):

    def default(self, obj):
    if isinstance(obj, datetime.datetime):
    return str(obj)

    return json.JSONEncoder.default(self, obj)

    while True:
    for name, dsn in DATABASE_DSNS.items():
    conn = psycopg2.connect(dsn)
    conn.autocommit = True

    cursor = conn.cursor(cursor_factory=DictCursor)

    cursor.execute("SELECT * FROM pg_locks, pg_stat_activity WHERE pg_locks.pid = pg_stat_activity.procpid;")

    with open(make_filename(name), 'w+') as output:
    run_time = str(datetime.datetime.now())

    for r in cursor:
    data = dict(r)
    data.update({
    'version': 1,
    'host': name,
    '@timestamp': datetime.datetime.utcnow().isoformat() + 'Z'
    })
    json_data = json.dumps(data, cls=DTEncoder)
    logger.info(json_data)
    cursor.close()
    conn.close()
    time.sleep(SLEEP_SECONDS)