Skip to content

Instantly share code, notes, and snippets.

@gdsaxton

gdsaxton/twitter-user-data.py

Last active December 16, 2018 19:46
Show Gist options
  • Save gdsaxton/9037367 to your computer and use it in GitHub Desktop.
Save gdsaxton/9037367 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. gdsaxton revised this gist Apr 26, 2018. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -48,7 +48,8 @@

    #INITIALIZE OUTPUT FILE AND WRITE HEADER ROW
    outfp = open(outfn, "w")
    outfp.write(string.join(fields, "\t") + "\n") # header
    #outfp.write(string.join(fields, "\t") + "\n") # header
    outfp.write("\t".join(fields) + "\n") # header

    #THE VARIABLE 'USERS' CONTAINS INFORMATION OF THE 32 TWITTER USER IDS LISTED ABOVE
    #THIS BLOCK WILL LOOP OVER EACH OF THESE IDS, CREATE VARIABLES, AND OUTPUT TO FILE
  2. gdsaxton revised this gist Apr 26, 2018. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -86,6 +86,7 @@
    for f in fields:
    lst.append(unicode(r[f]).replace("\/", "/"))
    #WRITE ROW WITH DATA IN LIST
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")
    #outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")
    outfp.write("\t".join(lst).encode('utf-8') + '\n')

    outfp.close()
  3. Gregory Saxton revised this gist May 12, 2014. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -83,7 +83,8 @@
    #CREATE EMPTY LIST
    lst = []
    #ADD DATA FOR EACH VARIABLE
    lst.append(unicode(r[f]).replace("\/", "/"))
    for f in fields:
    lst.append(unicode(r[f]).replace("\/", "/"))
    #WRITE ROW WITH DATA IN LIST
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")

  4. Gregory Saxton revised this gist May 12, 2014. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -83,7 +83,7 @@
    #CREATE EMPTY LIST
    lst = []
    #ADD DATA FOR EACH VARIABLE
    lst.append(unicode(r[f]).replace("\/", "/"))
    lst.append(unicode(r[f]).replace("\/", "/"))
    #WRITE ROW WITH DATA IN LIST
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")

  5. Gregory Saxton revised this gist Apr 8, 2014. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -39,7 +39,7 @@
    users = t.lookup_user(user_id = ids)

    #NAME OUR OUTPUT FILE - %i WILL BE REPLACED BY CURRENT MONTH, DAY, AND YEAR
    outfn = "friend_followers_twython_%i.%i.%i.txt" % (now.month, now.day, now.year)
    outfn = "twitter_user_data_%i.%i.%i.txt" % (now.month, now.day, now.year)

    #NAMES FOR HEADER ROW IN OUTPUT FILE
    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
  6. Gregory Saxton revised this gist Mar 12, 2014. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -82,9 +82,9 @@
    print r
    #CREATE EMPTY LIST
    lst = []
    for f in fields:
    #ADD DATA FOR EACH VARIABLE
    lst.append(unicode(r[f]).replace("\/", "/"))
    # PRINT lst
    #WRITE ROW WITH DATA IN LIST
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")

    outfp.close()
  7. Gregory Saxton revised this gist Mar 12, 2014. 1 changed file with 6 additions and 1 deletion.
    7 changes: 6 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -51,12 +51,15 @@
    outfp.write(string.join(fields, "\t") + "\n") # header

    #THE VARIABLE 'USERS' CONTAINS INFORMATION OF THE 32 TWITTER USER IDS LISTED ABOVE
    #THIS BLOCK WILL LOOP OVER EACH OF THESE IDS,
    #THIS BLOCK WILL LOOP OVER EACH OF THESE IDS, CREATE VARIABLES, AND OUTPUT TO FILE
    for entry in users:
    #CREATE EMPTY DICTIONARY
    r = {}
    for f in fields:
    r[f] = ""
    #ASSIGN VALUE OF 'ID' FIELD IN JSON TO 'ID' FIELD IN OUR DICTIONARY
    r['id'] = entry['id']
    #SAME WITH 'SCREEN_NAME' HERE, AND FOR REST OF THE VARIABLES
    r['screen_name'] = entry['screen_name']
    r['name'] = entry['name']
    r['created_at'] = entry['created_at']
    @@ -71,11 +74,13 @@
    r['protected'] = entry['protected']
    r['location'] = entry['location']
    r['lang'] = entry['lang']
    #NOT EVERY ID WILL HAVE A 'URL' KEY, SO CHECK FOR ITS EXISTENCE WITH IF CLAUSE
    if 'url' in entry['entities']:
    r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    r['expanded_url'] = ''
    print r
    #CREATE EMPTY LIST
    lst = []
    for f in fields:
    lst.append(unicode(r[f]).replace("\/", "/"))
  8. Gregory Saxton revised this gist Mar 12, 2014. 1 changed file with 11 additions and 3 deletions.
    14 changes: 11 additions & 3 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -12,15 +12,17 @@
    import sys
    import string
    import simplejson
    from twython import Twython

    #WE WILL USE THE VARIABLES DAY, MONTH, AND YEAR FOR OUR OUTPUT FILE NAME
    import datetime
    now = datetime.datetime.now()
    day=int(now.day)
    month=int(now.month)
    year=int(now.year)

    from twython import Twython

    #FOR OAUTH AUTHENTICATION -- NEEDED TO ACCESS THE TWITTER API
    t = Twython(app_key='APP_KEY', #REPLACE 'APP_KEY' WITH YOUR APP KEY, ETC., IN THE NEXT 4 LINES
    app_secret='APP_SECRET',
    oauth_token='OAUTH_TOKEN',
    @@ -32,18 +34,24 @@
    16315120, 16566133, 16686673, 16801671, 41900627, 42645839, 42731742, 44157002, 44988185,\
    48073289, 48827616, 49702654, 50310311, 50361094,"

    ##### NEW VERSION OF TWYTHON (6/26/2013) REQUIRES THE FOLLOWING:
    #ACCESS THE LOOKUP_USER METHOD OF THE TWITTER API -- GRAB INFO ON UP TO 100 IDS WITH EACH API CALL
    #THE VARIABLE USERS IS A JSON FILE WITH DATA ON THE 32 TWITTER USERS LISTED ABOVE
    users = t.lookup_user(user_id = ids)

    #NAME OUR OUTPUT FILE - %i WILL BE REPLACED BY CURRENT MONTH, DAY, AND YEAR
    outfn = "friend_followers_twython_%i.%i.%i.txt" % (now.month, now.day, now.year)

    #NAMES FOR HEADER ROW IN OUTPUT FILE
    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
    favourites_count listed_count \
    contributors_enabled description protected location lang expanded_url".split()


    #INITIALIZE OUTPUT FILE AND WRITE HEADER ROW
    outfp = open(outfn, "w")
    outfp.write(string.join(fields, "\t") + "\n") # header

    #THE VARIABLE 'USERS' CONTAINS INFORMATION OF THE 32 TWITTER USER IDS LISTED ABOVE
    #THIS BLOCK WILL LOOP OVER EACH OF THESE IDS,
    for entry in users:
    r = {}
    for f in fields:
  9. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 0 additions and 1 deletion.
    1 change: 0 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -10,7 +10,6 @@
    """

    import sys
    import urllib
    import string
    import simplejson

  10. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 1 addition and 2 deletions.
    3 changes: 1 addition & 2 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -5,7 +5,7 @@
    Use Twitter API to grab user information from list of organizations;
    export text file
    USES TWYTHON AND OAUTH
    Uses Twython module to access Twitter API
    """

    @@ -19,7 +19,6 @@
    day=int(now.day)
    month=int(now.month)
    year=int(now.year)
    hour=int(now.hour)

    from twython import Twython

  11. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 1 addition and 8 deletions.
    9 changes: 1 addition & 8 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -9,7 +9,6 @@
    """


    import sys
    import urllib
    import string
    @@ -28,20 +27,16 @@
    app_secret='APP_SECRET',
    oauth_token='OAUTH_TOKEN',
    oauth_token_secret='OAUTH_TOKEN_SECRET')



    #REPLACE WITH YOUR IDS
    #REPLACE WITH YOUR LIST OF TWITTER USER IDS
    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894,\
    15029174, 15474846, 15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542,\
    16315120, 16566133, 16686673, 16801671, 41900627, 42645839, 42731742, 44157002, 44988185,\
    48073289, 48827616, 49702654, 50310311, 50361094,"


    ##### NEW VERSION OF TWYTHON (6/26/2013) REQUIRES THE FOLLOWING:
    users = t.lookup_user(user_id = ids)


    outfn = "friend_followers_twython_%i.%i.%i.txt" % (now.month, now.day, now.year)

    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
    @@ -51,7 +46,6 @@
    outfp = open(outfn, "w")
    outfp.write(string.join(fields, "\t") + "\n") # header


    for entry in users:
    r = {}
    for f in fields:
    @@ -75,7 +69,6 @@
    r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    r['expanded_url'] = ''

    print r
    lst = []
    for f in fields:
  12. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 0 additions and 26 deletions.
    26 changes: 0 additions & 26 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -53,32 +53,6 @@


    for entry in users:
    screen_name = entry['screen_name']
    followers_count = entry['followers_count']
    friends_count = entry['friends_count']
    name = entry['name']
    created_at = entry['created_at']
    id = entry['id']
    url = entry['url']
    statuses_count = entry['statuses_count']
    favourites_count = entry['favourites_count']
    listed_count = entry['listed_count']
    contributors_enabled = entry['contributors_enabled']
    description = entry['description']
    protected = entry['protected']
    location = entry['location']
    lang = entry['lang']
    if 'url' in entry['entities']:
    expanded_url = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    expanded_url = ''
    print contributors_enabled
    print description
    print protected
    print location
    print lang
    print expanded_url
    print "ON TO NEXT USER..............."
    r = {}
    for f in fields:
    r[f] = ""
  13. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -2,7 +2,8 @@

    """
    Use Twitter API to grab user information from list of organizations; export text file
    Use Twitter API to grab user information from list of organizations;
    export text file
    USES TWYTHON AND OAUTH
  14. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 5 additions and 5 deletions.
    10 changes: 5 additions & 5 deletions twitter-user-data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -23,17 +23,17 @@

    from twython import Twython

    t = Twython(app_key='APP_KEY', #REPLACE 'APP_KEY' WITH YOUR APP KEY, ETC., IN THE NEXT 4 LINES
    t = Twython(app_key='APP_KEY', #REPLACE 'APP_KEY' WITH YOUR APP KEY, ETC., IN THE NEXT 4 LINES
    app_secret='APP_SECRET',
    oauth_token='OAUTH_TOKEN',
    oauth_token_secret='OAUTH_TOKEN_SECRET')



    #REPLACE WITH YOUR IDS
    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894, \
    15029174, 15474846, 15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542, \
    16315120, 16566133, 16686673, 16801671, 41900627, 42645839, 42731742, 44157002, 44988185, \
    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894,\
    15029174, 15474846, 15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542,\
    16315120, 16566133, 16686673, 16801671, 41900627, 42645839, 42731742, 44157002, 44988185,\
    48073289, 48827616, 49702654, 50310311, 50361094,"


    @@ -43,7 +43,7 @@

    outfn = "friend_followers_twython_%i.%i.%i.txt" % (now.month, now.day, now.year)

    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
    favourites_count listed_count \
    contributors_enabled description protected location lang expanded_url".split()

  15. Gregory Saxton renamed this gist Feb 16, 2014. 1 changed file with 0 additions and 0 deletions.
    0 Twitter User Data.py → twitter-user-data.py
    View file
    File renamed without changes.
  16. Gregory Saxton revised this gist Feb 16, 2014. 1 changed file with 15 additions and 26 deletions.
    41 changes: 15 additions & 26 deletions Twitter User Data.py
    View file
    Original file line number Diff line number Diff line change
    @@ -30,13 +30,11 @@




    #REPLACE WITH YOUR IDS
    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894, 15029174, 15474846, \
    15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542, 16315120, 16566133, 16686673, 16801671, \
    41900627, 42645839, 42731742, 44157002, 44988185, 48073289, 48827616, 49702654, 50310311, 50361094,"


    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894, \
    15029174, 15474846, 15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542, \
    16315120, 16566133, 16686673, 16801671, 41900627, 42645839, 42731742, 44157002, 44988185, \
    48073289, 48827616, 49702654, 50310311, 50361094,"


    ##### NEW VERSION OF TWYTHON (6/26/2013) REQUIRES THE FOLLOWING:
    @@ -54,8 +52,6 @@


    for entry in users:
    #print entry['screen_name'], entry['followers_count'], entry['friends_count'], entry['name'], entry['created_at'], entry['id'], entry['screen_name'], entry['url'], entry['statuses_count'], entry['favourites_count'], entry['listed_count']

    screen_name = entry['screen_name']
    followers_count = entry['followers_count']
    friends_count = entry['friends_count']
    @@ -66,12 +62,11 @@
    statuses_count = entry['statuses_count']
    favourites_count = entry['favourites_count']
    listed_count = entry['listed_count']
    contributors_enabled = entry['contributors_enabled'] #ADDED 7/19/13
    description = entry['description'] #ADDED 7/19/13
    protected = entry['protected'] #ADDED 7/19/13
    location = entry['location'] #ADDED 7/19/13
    lang = entry['lang'] #ADDED 7/19/13
    #display_url = entry['entities']['url']['urls'][0]['display_url']
    contributors_enabled = entry['contributors_enabled']
    description = entry['description']
    protected = entry['protected']
    location = entry['location']
    lang = entry['lang']
    if 'url' in entry['entities']:
    expanded_url = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    @@ -96,27 +91,21 @@
    r['statuses_count'] = entry['statuses_count']
    r['favourites_count'] = entry['favourites_count']
    r['listed_count'] = entry['listed_count']
    r['contributors_enabled'] = entry['contributors_enabled'] #ADDED 7/19/13
    r['description'] = entry['description'] #ADDED 7/19/13
    r['protected'] = entry['protected'] #ADDED 7/19/13
    r['location'] = entry['location'] #ADDED 7/19/13
    r['lang'] = entry['lang'] #ADDED 7/19/13
    #r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    r['contributors_enabled'] = entry['contributors_enabled']
    r['description'] = entry['description']
    r['protected'] = entry['protected']
    r['location'] = entry['location']
    r['lang'] = entry['lang']
    if 'url' in entry['entities']:
    r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    r['expanded_url'] = ''



    #r.update(entry)
    #r['description'] = en_txts #goes in after the update because "description" is a field
    print r

    lst = []
    for f in fields:
    lst.append(unicode(r[f]).replace("\/", "/"))
    # print lst
    # PRINT lst
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")

    outfp.close()
  17. Gregory Saxton renamed this gist Feb 16, 2014. 1 changed file with 0 additions and 0 deletions.
    0 Twitter User Data → Twitter User Data.py
    View file
    File renamed without changes.
  18. Gregory Saxton created this gist Feb 16, 2014.
    122 changes: 122 additions & 0 deletions Twitter User Data
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,122 @@
    #!/usr/bin/env python

    """
    Use Twitter API to grab user information from list of organizations; export text file
    USES TWYTHON AND OAUTH
    """


    import sys
    import urllib
    import string
    import simplejson

    import datetime
    now = datetime.datetime.now()
    day=int(now.day)
    month=int(now.month)
    year=int(now.year)
    hour=int(now.hour)

    from twython import Twython

    t = Twython(app_key='APP_KEY', #REPLACE 'APP_KEY' WITH YOUR APP KEY, ETC., IN THE NEXT 4 LINES
    app_secret='APP_SECRET',
    oauth_token='OAUTH_TOKEN',
    oauth_token_secret='OAUTH_TOKEN_SECRET')




    #REPLACE WITH YOUR IDS
    ids = "4816,9715012,13023422, 13393052, 14226882, 14235041, 14292458, 14335586, 14730894, 15029174, 15474846, \
    15634728, 15689319, 15782399, 15946841, 16116519, 16148677, 16223542, 16315120, 16566133, 16686673, 16801671, \
    41900627, 42645839, 42731742, 44157002, 44988185, 48073289, 48827616, 49702654, 50310311, 50361094,"




    ##### NEW VERSION OF TWYTHON (6/26/2013) REQUIRES THE FOLLOWING:
    users = t.lookup_user(user_id = ids)


    outfn = "friend_followers_twython_%i.%i.%i.txt" % (now.month, now.day, now.year)

    fields = "id screen_name name created_at url followers_count friends_count statuses_count \
    favourites_count listed_count \
    contributors_enabled description protected location lang expanded_url".split()

    outfp = open(outfn, "w")
    outfp.write(string.join(fields, "\t") + "\n") # header


    for entry in users:
    #print entry['screen_name'], entry['followers_count'], entry['friends_count'], entry['name'], entry['created_at'], entry['id'], entry['screen_name'], entry['url'], entry['statuses_count'], entry['favourites_count'], entry['listed_count']

    screen_name = entry['screen_name']
    followers_count = entry['followers_count']
    friends_count = entry['friends_count']
    name = entry['name']
    created_at = entry['created_at']
    id = entry['id']
    url = entry['url']
    statuses_count = entry['statuses_count']
    favourites_count = entry['favourites_count']
    listed_count = entry['listed_count']
    contributors_enabled = entry['contributors_enabled'] #ADDED 7/19/13
    description = entry['description'] #ADDED 7/19/13
    protected = entry['protected'] #ADDED 7/19/13
    location = entry['location'] #ADDED 7/19/13
    lang = entry['lang'] #ADDED 7/19/13
    #display_url = entry['entities']['url']['urls'][0]['display_url']
    if 'url' in entry['entities']:
    expanded_url = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    expanded_url = ''
    print contributors_enabled
    print description
    print protected
    print location
    print lang
    print expanded_url
    print "ON TO NEXT USER..............."
    r = {}
    for f in fields:
    r[f] = ""
    r['id'] = entry['id']
    r['screen_name'] = entry['screen_name']
    r['name'] = entry['name']
    r['created_at'] = entry['created_at']
    r['url'] = entry['url']
    r['followers_count'] = entry['followers_count']
    r['friends_count'] = entry['friends_count']
    r['statuses_count'] = entry['statuses_count']
    r['favourites_count'] = entry['favourites_count']
    r['listed_count'] = entry['listed_count']
    r['contributors_enabled'] = entry['contributors_enabled'] #ADDED 7/19/13
    r['description'] = entry['description'] #ADDED 7/19/13
    r['protected'] = entry['protected'] #ADDED 7/19/13
    r['location'] = entry['location'] #ADDED 7/19/13
    r['lang'] = entry['lang'] #ADDED 7/19/13
    #r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    if 'url' in entry['entities']:
    r['expanded_url'] = entry['entities']['url']['urls'][0]['expanded_url']
    else:
    r['expanded_url'] = ''



    #r.update(entry)
    #r['description'] = en_txts #goes in after the update because "description" is a field
    print r

    lst = []
    for f in fields:
    lst.append(unicode(r[f]).replace("\/", "/"))
    # print lst
    outfp.write(string.join(lst, "\t").encode("utf-8") + "\n")

    outfp.close()