Created
July 4, 2014 05:32
-
-
Save menghan/17a370f16e5fefd068f3 to your computer and use it in GitHub Desktop.
Revisions
-
menghan created this gist
Jul 4, 2014 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,253 @@ menghan@MenghanMac:~$ cat /tmp/diffs --- httpclient.py 2013-11-15 11:21:29.000000000 +0800 +++ httpclient.pyc_dis 2014-07-04 13:30:27.000000000 +0800 @@ -1,3 +1,4 @@ +#Embedded file name: /Users/menghan/.virtualenvs/common/lib/python2.7/site-packages/tornado/httpclient.py """Blocking and non-blocking HTTP client interfaces. This module defines a common interface shared by two implementations, @@ -26,20 +27,16 @@ you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum supported version is 7.18.2, and the recommended version is 7.21.1 or newer. """ - from __future__ import absolute_import, division, print_function, with_statement - import functools import time import weakref - from tornado.concurrent import Future from tornado.escape import utf8 from tornado import httputil, stack_context from tornado.ioloop import IOLoop from tornado.util import Configurable - class HTTPClient(object): """A blocking HTTP client. @@ -55,6 +52,7 @@ print "Error:", e http_client.close() """ + def __init__(self, async_client_class=None, **kwargs): self._io_loop = IOLoop() if async_client_class is None: @@ -81,8 +79,7 @@ If an error occurs during the fetch, we raise an `HTTPError`. """ - response = self._io_loop.run_sync(functools.partial( - self._async_client.fetch, request, **kwargs)) + response = self._io_loop.run_sync(functools.partial(self._async_client.fetch, request, **kwargs)) response.rethrow() return response @@ -110,6 +107,7 @@ deprecated. The implementation subclass as well as arguments to its constructor can be set with the static method `configure()` """ + @classmethod def configurable_base(cls): return AsyncHTTPClient @@ -130,8 +128,7 @@ io_loop = io_loop or IOLoop.current() if io_loop in cls._async_clients() and not force_instance: return cls._async_clients()[io_loop] - instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, - **kwargs) + instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, **kwargs) if not force_instance: cls._async_clients()[io_loop] = instance return instance @@ -169,9 +166,6 @@ """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) - # We may modify this (to add Host, Accept-Encoding, etc), - # so make sure we don't modify the caller's object. This is also - # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = Future() @@ -183,12 +177,11 @@ if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: - response = HTTPResponse( - request, 599, error=exc, - request_time=time.time() - request.start_time) + response = HTTPResponse(request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) + future.add_done_callback(handle_future) def handle_response(response): @@ -196,6 +189,7 @@ future.set_exception(response.error) else: future.set_result(response) + self.fetch_impl(request, handle_response) return future @@ -227,32 +221,9 @@ class HTTPRequest(object): """HTTP client request object.""" + _DEFAULTS = dict(connect_timeout=20.0, request_timeout=20.0, follow_redirects=True, max_redirects=5, use_gzip=True, proxy_password='', allow_nonstandard_methods=False, validate_cert=True) - # Default values for HTTPRequest parameters. - # Merged with the values on the request object by AsyncHTTPClient - # implementations. - _DEFAULTS = dict( - connect_timeout=20.0, - request_timeout=20.0, - follow_redirects=True, - max_redirects=5, - use_gzip=True, - proxy_password='', - allow_nonstandard_methods=False, - validate_cert=True) - - def __init__(self, url, method="GET", headers=None, body=None, - auth_username=None, auth_password=None, auth_mode=None, - connect_timeout=None, request_timeout=None, - if_modified_since=None, follow_redirects=None, - max_redirects=None, user_agent=None, use_gzip=None, - network_interface=None, streaming_callback=None, - header_callback=None, prepare_curl_callback=None, - proxy_host=None, proxy_port=None, proxy_username=None, - proxy_password=None, allow_nonstandard_methods=None, - validate_cert=None, ca_certs=None, - allow_ipv6=None, - client_key=None, client_cert=None): + def __init__(self, url, method = 'GET', headers = None, body = None, auth_username = None, auth_password = None, auth_mode = None, connect_timeout = None, request_timeout = None, if_modified_since = None, follow_redirects = None, max_redirects = None, user_agent = None, use_gzip = None, network_interface = None, streaming_callback = None, header_callback = None, prepare_curl_callback = None, proxy_host = None, proxy_port = None, proxy_username = None, proxy_password = None, allow_nonstandard_methods = None, validate_cert = None, ca_certs = None, allow_ipv6 = None, client_key = None, client_cert = None): r"""All parameters except ``url`` are optional. :arg string url: URL to fetch @@ -318,8 +289,7 @@ if headers is None: headers = httputil.HTTPHeaders() if if_modified_since: - headers["If-Modified-Since"] = httputil.format_timestamp( - if_modified_since) + headers['If-Modified-Since'] = httputil.format_timestamp(if_modified_since) self.proxy_host = proxy_host self.proxy_port = proxy_port self.proxy_username = proxy_username @@ -379,15 +349,14 @@ plus ``queue``, which is the delay (if any) introduced by waiting for a slot under `AsyncHTTPClient`'s ``max_clients`` setting. """ - def __init__(self, request, code, headers=None, buffer=None, - effective_url=None, error=None, request_time=None, - time_info=None, reason=None): + + def __init__(self, request, code, headers = None, buffer = None, effective_url = None, error = None, request_time = None, time_info = None, reason = None): if isinstance(request, _RequestProxy): self.request = request.request else: self.request = request self.code = code - self.reason = reason or httputil.responses.get(code, "Unknown") + self.reason = reason or httputil.responses.get(code, 'Unknown') if headers is not None: self.headers = headers else: @@ -410,10 +379,9 @@ def _get_body(self): if self.buffer is None: - return None - elif self._body is None: + return + if self._body is None: self._body = self.buffer.getvalue() - return self._body body = property(_get_body) @@ -424,8 +392,8 @@ raise self.error def __repr__(self): - args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) - return "%s(%s)" % (self.__class__.__name__, args) + args = ','.join(('%s=%r' % i for i in sorted(self.__dict__.items()))) + return '%s(%s)' % (self.__class__.__name__, args) class HTTPError(Exception): @@ -442,11 +410,12 @@ and you can look at ``error.response.headers['Location']`` to see the destination of the redirect. """ + def __init__(self, code, message=None, response=None): self.code = code - message = message or httputil.responses.get(code, "Unknown") + message = message or httputil.responses.get(code, 'Unknown') self.response = response - Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) + Exception.__init__(self, 'HTTP %d: %s' % (self.code, message)) class _RequestProxy(object): @@ -454,6 +423,7 @@ Used internally by AsyncHTTPClient implementations. """ + def __init__(self, request, defaults): self.request = request self.defaults = defaults @@ -465,33 +435,33 @@ elif self.defaults is not None: return self.defaults.get(name, None) else: - return None + return def main(): from tornado.options import define, options, parse_command_line - define("print_headers", type=bool, default=False) - define("print_body", type=bool, default=True) - define("follow_redirects", type=bool, default=True) - define("validate_cert", type=bool, default=True) + define('print_headers', type=bool, default=False) + define('print_body', type=bool, default=True) + define('follow_redirects', type=bool, default=True) + define('validate_cert', type=bool, default=True) args = parse_command_line() client = HTTPClient() for arg in args: try: - response = client.fetch(arg, - follow_redirects=options.follow_redirects, - validate_cert=options.validate_cert, - ) + response = client.fetch(arg, follow_redirects=options.follow_redirects, validate_cert=options.validate_cert) except HTTPError as e: if e.response is not None: response = e.response else: raise + if options.print_headers: print(response.headers) if options.print_body: print(response.body) + client.close() -if __name__ == "__main__": + +if __name__ == '__main__': main()