Last active
June 26, 2018 09:43
-
-
Save Tremus/e616729a920df34d371fc5ca18c1aef2 to your computer and use it in GitHub Desktop.
Ever find yourself scraping data on bad internet connection? Then you need to relaunch your whole sequence again, possibly from the start. Or maybe you shut and reopened your laptop. This is a simply retry wrapper
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from time import sleep | |
| import socket | |
| def internet_connection(host="8.8.8.8", port=53, timeout=3): | |
| """ | |
| Host: 8.8.8.8 (google-public-dns-a.google.com) | |
| OpenPort: 53/tcp | |
| Service: domain (DNS/TCP) | |
| Source: https://stackoverflow.com/questions/3764291/checking-network-connection | |
| """ | |
| try: | |
| socket.setdefaulttimeout(timeout) | |
| socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) | |
| return True | |
| except Exception as e: | |
| print(e) | |
| return False | |
| class handle_bad_connection(object): | |
| def __init__(self, func): | |
| self.func = func | |
| def __get__(self, instance, owner): | |
| self.instance = instance | |
| return self.__call__ | |
| def __call__(self, *args, **kwargs): | |
| """ | |
| On a bad internet connection, your function may return None. | |
| Returning None however, may be an intended part of your function. | |
| When your response = None, check the internet connection. | |
| """ | |
| response = self.func(self.instance, *args, **kwargs) | |
| if not response and not internet_connection(): | |
| response = self.wait_for_connection_and_retry(*args, **kwargs) | |
| return response | |
| def wait_for_connection_and_retry(self, *args, **kwargs): | |
| print ('\nConnection error. Retrying...!\n') | |
| while not internet_connection(): | |
| sleep(10) | |
| print ('\nSuccessully reconnected!\n') | |
| return self.func(self, *args, **kwargs) | |
| # Use case | |
| class SuperMegaScraper(MegaScraper) | |
| @handle_bad_connection | |
| def super_scrape(self): | |
| # ... | |
| pass |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment