Skip to content

Instantly share code, notes, and snippets.

@SavvyGuard

SavvyGuard/dwq.py

Last active August 29, 2015 14:10
Show Gist options
  • Save SavvyGuard/8ca917d04b1bf94312a2 to your computer and use it in GitHub Desktop.
Save SavvyGuard/8ca917d04b1bf94312a2 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. SavvyGuard revised this gist Apr 23, 2015. 1 changed file with 14 additions and 11 deletions.
    25 changes: 14 additions & 11 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,12 +1,12 @@
    # /usr/bin/python

    # /usr/bin/python
    import subprocess
    import argparse
    import json
    import json
    import datetime

    def query(user, host, query):
    bash_query = '/opt/spark/bin/beeline -u jdbc:hive2://cdh-head-node-001:10000 -e "{query}" --silent --outputFormat=vertical'.format(query=query)

    bash_query = '/opt/spark/bin/beeline -u jdbc:hive2://dp-cluster-master-node-001:10000 -e "{query}" --silent --outputFormat=vertical'.format(query=query)
    process = subprocess.Popen(
    [
    'ssh',
    @@ -25,7 +25,7 @@ def query(user, host, query):
    try:
    clean_results[result_list[0]].extend(result_list[1:])
    except KeyError as e:
    clean_results[result_list[0]] = result_list[1:]
    clean_results[result_list[0]] = result_list[1:]

    result_string = json.dumps(clean_results,
    {
    @@ -36,7 +36,10 @@ def query(user, host, query):
    indent=4,
    separators=(',', ': ')
    )
    print result_string
    print result_string

    #store_in_firebase(clean_results)
    return clean_results

    def execute():
    parser = argparse.ArgumentParser()
    @@ -46,10 +49,10 @@ def execute():
    parser.add_argument("-q", "--query", help="set the query to run")

    args = parser.parse_args()

    query(args.user, args.host, args.query)


    if __name__ == '__main__':
    execute()

    if __name__ == '__main__':
    execute()

    #query('james','54.174.83.219', 'select count(1) as count from event.bid_request where ds=20141122 and h=9')
  2. SavvyGuard revised this gist Nov 22, 2014. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -23,9 +23,9 @@ def query(user, host, query):
    for result in filter(lambda x:x != '\n', raw[0:-1]):
    result_list = result.split()
    try:
    clean_results[result_list[0]].append(result_list[1:])
    clean_results[result_list[0]].extend(result_list[1:])
    except KeyError as e:
    clean_results[result_list[0]] = result_list[1:]
    clean_results[result_list[0]] = result_list[1:]

    result_string = json.dumps(clean_results,
    {
  3. SavvyGuard revised this gist Nov 22, 2014. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    # /usr/bin/python

    import subprocess
    import argparse
    import json
  4. SavvyGuard revised this gist Nov 22, 2014. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -15,7 +15,7 @@ def query(user, host, query):
    stderr=subprocess.PIPE
    )
    process.wait()
    raw = process.stdout.readlines()
    raw = process.stdout.readlines()
    cleaned_results = filter(lambda x:x != '\n', raw[0:-1])
    clean_results = {}
    for result in filter(lambda x:x != '\n', raw[0:-1]):
  5. SavvyGuard revised this gist Nov 22, 2014. 1 changed file with 20 additions and 4 deletions.
    24 changes: 20 additions & 4 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -15,10 +15,26 @@ def query(user, host, query):
    stderr=subprocess.PIPE
    )
    process.wait()
    raw = process.stdout.readlines()
    results = {result.split()[0] : result.split()[1:] for result in raw[0:-1]}
    print json.dumps(results)
    return results
    raw = process.stdout.readlines()
    cleaned_results = filter(lambda x:x != '\n', raw[0:-1])
    clean_results = {}
    for result in filter(lambda x:x != '\n', raw[0:-1]):
    result_list = result.split()
    try:
    clean_results[result_list[0]].append(result_list[1:])
    except KeyError as e:
    clean_results[result_list[0]] = result_list[1:]

    result_string = json.dumps(clean_results,
    {
    '4': 5,
    '6': 7
    },
    sort_keys=True,
    indent=4,
    separators=(',', ': ')
    )
    print result_string

    def execute():
    parser = argparse.ArgumentParser()
  6. SavvyGuard revised this gist Nov 22, 2014. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    import subprocess
    import argparse
    import ConfigParser
    import json

    def query(user, host, query):
    bash_query = '/opt/spark/bin/beeline -u jdbc:hive2://cdh-head-node-001:10000 -e "{query}" --silent --outputFormat=vertical'.format(query=query)
    @@ -17,7 +17,7 @@ def query(user, host, query):
    process.wait()
    raw = process.stdout.readlines()
    results = {result.split()[0] : result.split()[1:] for result in raw[0:-1]}
    print results
    print json.dumps(results)
    return results

    def execute():
  7. SavvyGuard created this gist Nov 22, 2014.
    37 changes: 37 additions & 0 deletions dwq.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,37 @@
    import subprocess
    import argparse
    import ConfigParser

    def query(user, host, query):
    bash_query = '/opt/spark/bin/beeline -u jdbc:hive2://cdh-head-node-001:10000 -e "{query}" --silent --outputFormat=vertical'.format(query=query)

    process = subprocess.Popen(
    [
    'ssh',
    '{user}@{host}'.format(user=user,host=host),
    '{bash_query}'.format(bash_query=bash_query)
    ],
    stdout=subprocess.PIPE,
    stderr=subprocess.PIPE
    )
    process.wait()
    raw = process.stdout.readlines()
    results = {result.split()[0] : result.split()[1:] for result in raw[0:-1]}
    print results
    return results

    def execute():
    parser = argparse.ArgumentParser()

    parser.add_argument("-u", "--user", help="set username for beeline machine")
    parser.add_argument("-t", "--host", help="set host for beeline machine")
    parser.add_argument("-q", "--query", help="set the query to run")

    args = parser.parse_args()

    query(args.user, args.host, args.query)


    if __name__ == '__main__':
    execute()