phi-line · February 23, 2019 00:08 · Feb 23, 2019 · Feb 23, 2019
diff --git a/gistfile1.txt → scraper.py b/gistfile1.txt → scraper.py
diff --git a/gistfile1.txt b/gistfile1.txt
@@ -0,0 +1,61 @@
+import asyncio
+import pyppeteer as pyp
+from pyppeteer import launch
+from bs4 import BeautifulSoup
+
+async def main():
+    browser = await launch(
+        headless=True,
+        devtools=True,
+        autoClose=False,
+    )
+
+    url = 'https://duckduckgo.com/'
+
+    page = await browser.newPage()
+
+    await page.goto(url)
+
+    await page.type(selector='#search_form_input_homepage',
+                    text='cats')
+
+    await asyncio.gather(
+        page.click(
+            "#search_button_homepage",
+            options={"waitUntil": "networkidle0"},
+        ),
+        page.waitForNavigation(),
+    )
+
+    page_source = await page.evaluate(
+        "new XMLSerializer().serializeToString(document);"
+    )
+
+    soup = BeautifulSoup(page_source, "html.parser")
+
+    links = soup.find_all('a', {'class': 'result__a'})
+    for link in links:
+        await spider(browser, link['href'])
+
+
+async def spider(browser, link):
+    try:
+        page = await browser.newPage()
+
+        await page.goto(link)
+
+        page_source = await page.evaluate(
+            "new XMLSerializer().serializeToString(document);"
+        )
+
+        soup = BeautifulSoup(page_source, "html.parser")
+
+        links = soup.find_all('a')
+        for link in links:
+            await spider(browser, link['href'])
+    except pyp.errors.NetworkError:
+        return
+
+
+if __name__ == '__main__':
+    asyncio.get_event_loop().run_until_complete(main())