Skip to content

Instantly share code, notes, and snippets.

@ChuuMong

ChuuMong/CurrencyCrawling.kt

Last active February 24, 2020 02:10
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ChuuMong/9dba16d59fe30e4b9793a6ad3f127b56 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ChuuMong/9dba16d59fe30e4b9793a6ad3f127b56 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. ChuuMong revised this gist Feb 24, 2020. 1 changed file with 5 additions and 0 deletions.
    5 changes: 5 additions & 0 deletions CurrencyCrawling.kt
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,8 @@
    import com.gargoylesoftware.htmlunit.WebClient
    import com.gargoylesoftware.htmlunit.html.HtmlDivision
    import com.gargoylesoftware.htmlunit.html.HtmlPage
    import org.jsoup.Jsoup
    import kotlin.system.measureTimeMillis

    data class Currency(val code: String, val engName: String)

  2. ChuuMong created this gist Feb 24, 2020.
    43 changes: 43 additions & 0 deletions CurrencyCrawling.kt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,43 @@

    data class Currency(val code: String, val engName: String)

    fun main() {
    val currencies = mutableListOf<Currency>()

    val time = measureTimeMillis {
    val webClient = WebClient()
    val htmlPage = webClient.getPage<HtmlPage>("https://widget-yahoo.ofx.com/")
    webClient.waitForBackgroundJavaScript(5 * 60 * 1000)
    val htmlDivisions = htmlPage.getByXPath<HtmlDivision>("//div[contains(@class, 'currency-option')]")
    htmlDivisions.forEach { div ->
    val spans = div.getElementsByTagName("span")
    val currencyCode = spans.find { span -> span.getAttribute("class") == "currency-option-code" }?.textContent
    val currencyEngName =
    spans.find { span -> span.getAttribute("class") == "currency-option-description" }?.textContent

    if (!currencyCode.isNullOrEmpty() && !currencyEngName.isNullOrEmpty()) {
    currencies.add(Currency(currencyCode, currencyEngName))
    }
    }

    println("total currency size : ${currencies.size}\n\n")

    currencies.forEach {
    if (it.code == "USD") {
    return
    }

    val currencyMarketUrl = getCurrencyInfoUrl(it.code)
    val doc = Jsoup.connect(currencyMarketUrl).get()
    val elem = doc.select("div[data-reactid=\"33\"]").select("span[data-reactid=\"34\"]")
    println("${it.code} - ${elem.text()}")
    }
    }

    println(time)
    }

    fun getCurrencyInfoUrl(currency: String): String {
    return "https://finance.yahoo.com/quote/USD$currency%3DX?p=USD$currency%3DX"
    }