Skip to content

Instantly share code, notes, and snippets.

@pshapiro

pshapiro/hitwise-audienceview-keyword-research.py

Created June 20, 2019 19:44
Show Gist options
  • Save pshapiro/04f659d3ffadfa136e27fca7187309d0 to your computer and use it in GitHub Desktop.
Save pshapiro/04f659d3ffadfa136e27fca7187309d0 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. pshapiro created this gist Jun 20, 2019.
    111 changes: 111 additions & 0 deletions hitwise-audienceview-keyword-research.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,111 @@
    from selenium import webdriver
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.common.action_chains import ActionChains
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.support.ui import Select
    import time
    import pandas as pd
    import os

    keyword = [
    'healthy eating',
    'healthy recipes',
    'keto diet',
    ]

    download_folder = r"C:\Users\paul.Shapiro\AudienceView"
    output_file = r"C:\Users\paul.Shapiro\AudienceView\keyword_profiles.csv"
    usern = ""
    passw = ""

    def get_keyword_profile(keyword):
    print("Starting process for " + keyword)
    time.sleep(10)
    print("Waiting 10 seconds")
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(1) > button > div > ng-include > svg > path').click()
    driver.find_element_by_css_selector('#editor > crosstab-drop-area:nth-child(3) > div > div.crosstab-drop-area.dropAreaExpanded > ul > div > div > li > div.adr-notification-seg-element-border.ng-scope > div > img').click()
    print("Click edit button")
    time.sleep(2)
    driver.find_element_by_css_selector('#new-crosstab > string-lightbox > div > div > div.bottomFrame > div > div > div > p').click()
    search = driver.find_element_by_id("strsearchinput")
    print("entering keyword: " + keyword)
    search.send_keys(keyword)
    print("wait 10 seconds")
    time.sleep(10)
    print("Find and click exact match keyword: " + keyword)
    driver.find_element_by_xpath("//*[text()='" + keyword + "']").click()
    driver.maximize_window()
    driver.find_element_by_css_selector('#new-crosstab > string-lightbox > div > div > div.optionsFrame.container-fluid > div > span > button').click()
    print("Generate new report")
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(2) > button > div > ng-include > svg').click()
    print("Wait 60 seconds")
    time.sleep(60)
    print("Iniate export")
    try:
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(5) > button > div > ng-include > svg > path').click()
    except e:
    print("Waiting another 60 seconds")
    time.sleep(60)
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(5) > button > div > ng-include > svg > path').click()
    print("Wait 2 seconds")
    time.sleep(2)
    print("Find select box")
    select = Select(driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-body.ng-scope > form > div:nth-child(1) > select'))
    print("Select CSV")
    select.select_by_value('1')
    print("Click export button")
    driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-footer.ng-scope > button.ssda-button.ssda-button-orange').click()
    print("Wait for text to appear")
    print("Wait 20 seconds")
    time.sleep(20)
    print("Downloading")
    driver.find_element_by_xpath('//a[@download]').click()
    print("Waiting 2 seconds")
    time.sleep(2)
    print("Clicking cancel")
    driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-footer.ng-scope > button.ssda-button.ssda-button-default').click()
    print("Waiting 10 seconds")
    time.sleep(10)
    print("Download complete for " + keyword)

    chrome_options = webdriver.ChromeOptions()
    prefs = {'download.default_directory' : download_folder }
    # chrome_options.add_argument("--headless")
    chrome_options.add_experimental_option('prefs', prefs)

    driver = webdriver.Chrome(chrome_options=chrome_options)
    driver.get("https://audience-view.com/")
    print("Launching login page.")
    username = driver.find_element_by_name("emailID")
    username.clear()
    username.send_keys(usern)
    print("Inserting username")
    password = driver.find_element_by_name("password")
    password.clear()
    password.send_keys(passw)
    print("Inserting password")
    password.send_keys(Keys.RETURN)
    print("Logging in")
    print("Waiting 2 seconds")
    time.sleep(2)
    print("Going to crosstab")
    driver.get("https://audience-view.com/#/crosstab/32617?tab=0")

    for x in keyword:
    get_keyword_profile(x)

    driver.close()

    last_column = 0
    df = pd.read_csv(download_folder + "/crosstab.csv", usecols = [*range(0, 4)])
    for file in os.listdir("./"):
    if file.endswith(".csv"):
    if not file.endswith("crosstab.csv"):
    if last_column > 0:
    df = merge
    new = pd.read_csv(file, usecols=[3])
    merge = pd.concat([df, new], axis=1)
    last_column += 1
    merge.to_csv(output_file)