Created
March 22, 2016 13:40
-
-
Save dantangfan/0c7cc6e68664a5018888 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # coding:utf-8 | |
| __all__ = ('RetryAsyncHTTPClient', 'RequestException') | |
| from tornado.httpclient import AsyncHTTPClient | |
| from tornado.gen import coroutine, Return | |
| from tornado import gen | |
| class RequestException(Exception): | |
| pass | |
| class RetryRequestException(RequestException): | |
| pass | |
| class FailedRequestException(RequestException): | |
| pass | |
| class RetryAsyncHTTPClient(object): | |
| """ | |
| Usage: | |
| @coroutine | |
| def my_client(): | |
| retry_client = RetryAsyncHTTPClient() | |
| try: | |
| response = yield retry_client.fetch("http://www.baidu.com") | |
| except FailedRequestException as e: | |
| pass | |
| """ | |
| RETRY_ERROR_CODE = (400, 500) | |
| def __init__(self, http_client=None, max_retry=3, retry_timeout=0.5): | |
| """ | |
| :param http_client: 可以自定义client | |
| :param max_retry: 最大尝试次数 | |
| :param retry_timeout: 两次请求之间的时间间隔 | |
| :return: | |
| """ | |
| self.http_client = http_client if http_client else AsyncHTTPClient() | |
| self.max_retry = max_retry | |
| self.retry_timeout = retry_timeout | |
| @coroutine | |
| def _do_fetch(self, request, callback, raise_error, **kwargs): | |
| try: | |
| response = yield self.http_client.fetch(request, callback, raise_error, **kwargs) | |
| except Exception as e: | |
| raise RetryRequestException(e) | |
| else: | |
| raise Return(response) | |
| @coroutine | |
| def fetch(self, request, callback=None, raise_error=True, **kwargs): | |
| attempt = 0 | |
| while True: | |
| try: | |
| response = yield self._do_fetch(request, callback, raise_error, **kwargs) | |
| except RequestException as e: | |
| attempt += 1 | |
| if attempt > self.max_retry: | |
| raise FailedRequestException("%s request failed, due to: %s" % (self.max_retry, e)) | |
| yield gen.sleep(self.retry_timeout) | |
| except Exception as e: | |
| raise FailedRequestException("Failed due to: %s" % e) | |
| else: | |
| raise Return(response) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment