Skip to content

Instantly share code, notes, and snippets.

@icook

icook/coinbase-scrape.py

Last active October 21, 2018 09:12
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save icook/d3069a2dfa12421bf56a2e2169df206f to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save icook/d3069a2dfa12421bf56a2e2169df206f to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. icook revised this gist Oct 21, 2018. 1 changed file with 8 additions and 1 deletion.
    9 changes: 8 additions & 1 deletion coinbase-scrape.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,7 @@
    import requests
    import pprint
    import time
    import csv
    from datetime import datetime
    from dateutil.relativedelta import relativedelta

    @@ -24,4 +25,10 @@
    print("Got {} entries from {} to {}".format(len(processed), start, end))
    entries.extend(processed)
    grab_time = grab_time + chunk_size
    time.sleep(0.35) # They allow 3 req/s. Sloppy
    time.sleep(0.35) # They allow 3 req/s. Sloppy

    with open('coinbase-3year.csv', 'w') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(['open', 'high', 'low', 'close', 'volume', 'datetime', 'timestamp'])
    for e in entries:
    writer.writerow([e['open'], e['high'], e['low'], e['close'], e['volume'], datetime.utcfromtimestamp(e['ts']), e['ts']])
  2. icook revised this gist Oct 21, 2018. No changes.
  3. icook created this gist Oct 21, 2018.
    27 changes: 27 additions & 0 deletions coinbase-scrape.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,27 @@
    import requests
    import pprint
    import time
    from datetime import datetime
    from dateutil.relativedelta import relativedelta

    entries = []
    granularity = 900
    chunk_size = relativedelta(seconds=granularity * 300)
    grab_time = datetime.utcnow() - relativedelta(years=3)
    while 1:
    # Our loop exit condition
    if grab_time > datetime.utcnow():
    break
    # Get the coinbase entries
    start = grab_time.isoformat()
    end = (grab_time + chunk_size).isoformat()
    req = "https://api.pro.coinbase.com/products/BTC-USD/candles?granularity={}&start={}&end={}".format(
    granularity, start, end)
    print(req)
    res = requests.get(req)
    # Convert the output to our universal format
    processed = [{"ts": r[0], "open": r[1], "high": r[2], "low": r[3], "close": r[4], "volume": r[5], "exchange": "coinbase"} for r in res.json()]
    print("Got {} entries from {} to {}".format(len(processed), start, end))
    entries.extend(processed)
    grab_time = grab_time + chunk_size
    time.sleep(0.35) # They allow 3 req/s. Sloppy