#!/usr/bin/env python # coding:utf-8 __all__ = ('RetryAsyncHTTPClient', 'RequestException') from tornado.httpclient import AsyncHTTPClient from tornado.gen import coroutine, Return from tornado import gen class RequestException(Exception): pass class RetryRequestException(RequestException): pass class FailedRequestException(RequestException): pass class RetryAsyncHTTPClient(object): """ Usage: @coroutine def my_client(): retry_client = RetryAsyncHTTPClient() try: response = yield retry_client.fetch("http://www.baidu.com") except FailedRequestException as e: pass """ RETRY_ERROR_CODE = (400, 500) def __init__(self, http_client=None, max_retry=3, retry_timeout=0.5): """ :param http_client: 可以自定义client :param max_retry: 最大尝试次数 :param retry_timeout: 两次请求之间的时间间隔 :return: """ self.http_client = http_client if http_client else AsyncHTTPClient() self.max_retry = max_retry self.retry_timeout = retry_timeout @coroutine def _do_fetch(self, request, callback, raise_error, **kwargs): try: response = yield self.http_client.fetch(request, callback, raise_error, **kwargs) except Exception as e: raise RetryRequestException(e) else: raise Return(response) @coroutine def fetch(self, request, callback=None, raise_error=True, **kwargs): attempt = 0 while True: try: response = yield self._do_fetch(request, callback, raise_error, **kwargs) except RequestException as e: attempt += 1 if attempt > self.max_retry: raise FailedRequestException("%s request failed, due to: %s" % (self.max_retry, e)) yield gen.sleep(self.retry_timeout) except Exception as e: raise FailedRequestException("Failed due to: %s" % e) else: raise Return(response)