Skip to content

Instantly share code, notes, and snippets.

@mike-luabase

mike-luabase/luabase_api.ipynb

Last active August 29, 2022 13:27
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save mike-luabase/faf1058d361e17548f968d21b6d5e73c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save mike-luabase/faf1058d361e17548f968d21b6d5e73c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. mike-luabase revised this gist Aug 29, 2022. No changes.
  2. mike-luabase revised this gist Aug 29, 2022. 1 changed file with 9 additions and 0 deletions.
    9 changes: 9 additions & 0 deletions luabase_api.ipynb
    View file
    Original file line number Diff line number Diff line change
    @@ -2212,6 +2212,15 @@
    "data"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
    "LUABASE_API_KEY"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": null,
  3. mike-luabase revised this gist Aug 29, 2022. 1 changed file with 1972 additions and 1 deletion.
    1,973 changes: 1,972 additions & 1 deletion luabase_api.ipynb
    View file
    1,972 additions, 1 deletion not shown because the diff is too large. Please use a local Git client to view these changes.
  4. mike-luabase revised this gist Aug 29, 2022. 1 changed file with 90 additions and 0 deletions.
    90 changes: 90 additions & 0 deletions luabase_api.ipynb
    View file
    Original file line number Diff line number Diff line change
    @@ -157,6 +157,96 @@
    " res = get_table_days(table, 3, 'luabase-not-public', '8e86ae3479be49bfb5fdc843c6c6838f')\n",
    " print(res)"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 9,
    "metadata": {},
    "outputs": [
    {
    "name": "stdout",
    "output_type": "stream",
    "text": [
    "sql: \n",
    "select *\n",
    "from ethereum.blocks as t\n",
    "where toYYYYMMDD(timestamp) = 20220801\n",
    "\n",
    "done: 20220801 <Response [422]>\n"
    ]
    }
    ],
    "source": [
    "url = 'https://q.luabase.com/run'\n",
    "bucket_name = 'lua-test-eth-full-sync'\n",
    "table_dict = {\n",
    " 'name': 'blocks',\n",
    " 'dt': 'timestamp',\n",
    "}\n",
    "table = table_dict['name']\n",
    "table_dt = table_dict['dt']\n",
    "integration_id = '8e86ae3479be49bfb5fdc843c6c6838f'\n",
    "ddt = 20220801\n",
    "sql = f'''\n",
    "select *\n",
    "from ethereum.{table} as t\n",
    "where toYYYYMMDD({table_dt}) = {ddt}\n",
    "'''\n",
    "print('sql: ', sql)\n",
    "fname = f'{table}_{ddt}'\n",
    "\n",
    "\n",
    "payload = {\n",
    " 'block': {\n",
    " 'data_uuid': fname,\n",
    " 'details': {\n",
    " 'sql': sql,\n",
    " 'destination': {\n",
    " \"type\": \"s3\",\n",
    " \"paused\": False,\n",
    " \"fileName\": fname,\n",
    " \"ifExists\": \"replace\",\n",
    " \"bucketName\": bucket_name,\n",
    " \"integrationId\": integration_id,\n",
    " },\n",
    " },\n",
    " },\n",
    " 'api_key': LUABASE_API_KEY,\n",
    "}\n",
    "headers = {'content-type': 'application/json'}\n",
    "response = requests.request('POST', url, json=payload, headers=headers)\n",
    "data = response.json()\n",
    "print('done: ', ddt, response)"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 10,
    "metadata": {},
    "outputs": [
    {
    "data": {
    "text/plain": [
    "{'detail': [{'loc': ['body', 'api_key'],\n",
    " 'msg': 'field required',\n",
    " 'type': 'value_error.missing'}]}"
    ]
    },
    "execution_count": 10,
    "metadata": {},
    "output_type": "execute_result"
    }
    ],
    "source": [
    "data"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": []
    }
    ],
    "metadata": {
  5. mike-luabase revised this gist Aug 29, 2022. 1 changed file with 74 additions and 3 deletions.
    77 changes: 74 additions & 3 deletions luabase_api.ipynb
    View file
    Original file line number Diff line number Diff line change
    @@ -44,7 +44,7 @@
    },
    {
    "cell_type": "code",
    "execution_count": 4,
    "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
    @@ -78,14 +78,85 @@
    },
    {
    "cell_type": "code",
    "execution_count": null,
    "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "outputs": [
    {
    "name": "stdout",
    "output_type": "stream",
    "text": [
    "gettting dates for blocks\n",
    "gettting dates for transactions\n",
    "gettting dates for logs\n",
    "gettting dates for events\n"
    ]
    }
    ],
    "source": [
    "for table in tables:\n",
    " print('gettting dates for', table['name'])\n",
    " table['dates_to_get'] = get_table_dates(table)"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
    "def get_table_days(table_dict, num_days_to_get, bucket_name, integration_id):\n",
    " url = 'https://q.luabase.com/run'\n",
    " # https://q.luabase.com/run/async\n",
    "\n",
    " table = table_dict['name']\n",
    " table_dt = table_dict['dt']\n",
    " # start at 1 to remove incomplete current day\n",
    " for row in table_dict['dates_to_get']['data'][1:num_days_to_get]:\n",
    " ddt = row['ddt']\n",
    " sql = f'''\n",
    " select *\n",
    " from ethereum.{table} as t\n",
    " where toYYYYMMDD({table_dt}) = {ddt}\n",
    " '''\n",
    " print('sql: ', sql)\n",
    " fname = f'{table}_{ddt}'\n",
    "\n",
    " payload = {\n",
    " 'block': {\n",
    " 'data_uuid': fname,\n",
    " 'details': {\n",
    " 'sql': sql,\n",
    " 'destination': {\n",
    " \"type\": \"s3\",\n",
    " \"paused\": False,\n",
    " \"fileName\": fname,\n",
    " \"ifExists\": \"replace\",\n",
    " \"bucketName\": bucket_name,\n",
    " \"integrationId\": integration_id,\n",
    " },\n",
    " },\n",
    " },\n",
    " 'api_key': LUABASE_API_KEY,\n",
    " }\n",
    " headers = {'content-type': 'application/json'}\n",
    " response = requests.request('POST', url, json=payload, headers=headers)\n",
    " data = response.json()\n",
    " print('done: ', ddt, response)\n",
    " return data"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
    "# lua-test-eth-full-sync\n",
    "\n",
    "for table in tables:\n",
    " res = get_table_days(table, 3, 'luabase-not-public', '8e86ae3479be49bfb5fdc843c6c6838f')\n",
    " print(res)"
    ]
    }
    ],
    "metadata": {
  6. mike-luabase revised this gist Aug 29, 2022. 1 changed file with 118 additions and 1 deletion.
    119 changes: 118 additions & 1 deletion luabase_api.ipynb
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1,118 @@
    ‎‎​
    {
    "cells": [
    {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "LUABASE_API_KEY = os.getenv('LUABASE_USER_API_KEY')"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
    "tables = [\n",
    " {\n",
    " 'name': 'blocks',\n",
    " 'dt': 'timestamp',\n",
    " },\n",
    " {\n",
    " 'name': 'transactions',\n",
    " 'dt': 'block_timestamp',\n",
    " },\n",
    " {\n",
    " 'name': 'logs',\n",
    " 'dt': 'block_timestamp',\n",
    " },\n",
    " {\n",
    " 'name': 'events',\n",
    " 'dt': 'block_timestamp',\n",
    " },\n",
    "]"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
    "def get_table_dates(table_dict):\n",
    " url = 'https://q.luabase.com/run'\n",
    " table = table_dict['name']\n",
    " table_dt = table_dict['dt']\n",
    " sql = f'''\n",
    " select \n",
    " toYYYYMM({table_dt}) as mdt,\n",
    " toYYYYMMDD({table_dt}) as ddt,\n",
    " count() as ct\n",
    " from ethereuam.{table} as t\n",
    " group by ddt, mdt\n",
    " order by ddt desc\n",
    " '''\n",
    "\n",
    " payload = {\n",
    " 'block': {\n",
    " 'details': {\n",
    " 'sql': sql,\n",
    " },\n",
    " },\n",
    " 'api_key': LUABASE_API_KEY,\n",
    " }\n",
    " headers = {'content-type': 'application/json'}\n",
    " response = requests.request('POST', url, json=payload, headers=headers)\n",
    " data = response.json()\n",
    " return data"
    ]
    },
    {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
    "for table in tables:\n",
    " print('gettting dates for', table['name'])\n",
    " table['dates_to_get'] = get_table_dates(table)"
    ]
    }
    ],
    "metadata": {
    "kernelspec": {
    "display_name": "Python 3.9.7 ('luabase')",
    "language": "python",
    "name": "python3"
    },
    "language_info": {
    "codemirror_mode": {
    "name": "ipython",
    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.9.7"
    },
    "orig_nbformat": 4,
    "vscode": {
    "interpreter": {
    "hash": "53271a145c95c72be6fee3a8e5453a952d89c15933608c3f0c680e700a2ae5bf"
    }
    }
    },
    "nbformat": 4,
    "nbformat_minor": 2
    }
  7. mike-luabase created this gist Aug 29, 2022.
    1 change: 1 addition & 0 deletions luabase_api.ipynb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1 @@
    ‎‎​