Skip to content

Instantly share code, notes, and snippets.

@m7catsue

m7catsue/download_multiple.py

Forked from Hammer2900/download_multiple.py
Created March 23, 2017 07:30
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save m7catsue/c4ac72f677251cc053d99c204562d6cf to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save m7catsue/c4ac72f677251cc053d99c204562d6cf to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @harrisont harrisont revised this gist Jun 6, 2016. 1 changed file with 60 additions and 0 deletions.
    60 changes: 60 additions & 0 deletions download_multiple_sending_chunks_to_sink.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,60 @@
    import asyncio
    from contextlib import closing

    import aiohttp


    CHUNK_SIZE = 4 * 1024 # 4 KB


    def coroutine(func):
    def start(*args, **kwargs):
    cr = func(*args, **kwargs)
    next(cr)
    return cr
    return start


    @coroutine
    def chunk_printer(url: str):
    while True:
    chunk = yield
    print('got chunk: {}: {} bytes'.format(url, len(chunk)))


    async def download_file(session: aiohttp.ClientSession, url: str, sink):
    async with session.get(url) as response:
    assert response.status == 200
    while True:
    chunk = await response.content.read(CHUNK_SIZE)
    if not chunk:
    break
    sink.send(chunk)
    return url


    @asyncio.coroutine
    def download_multiple(session: aiohttp.ClientSession):
    urls = (
    'http://cnn.com',
    'http://nytimes.com',
    'http://google.com',
    'http://leagueoflegends.com',
    'http://python.org',
    )
    download_futures = [download_file(session, url, sink=chunk_printer(url)) for url in urls]
    print('Results')
    for download_future in asyncio.as_completed(download_futures):
    result = yield from download_future
    print('finished:', result)
    return urls


    def main():
    with closing(asyncio.get_event_loop()) as loop:
    with aiohttp.ClientSession() as session:
    result = loop.run_until_complete(download_multiple(session))
    print('finished:', result)


    main()
  2. @harrisont harrisont revised this gist Jun 6, 2016. 1 changed file with 4 additions and 1 deletion.
    5 changes: 4 additions & 1 deletion download_multiple.py
    View file
    Original file line number Diff line number Diff line change
    @@ -14,8 +14,11 @@ async def download_file(session: aiohttp.ClientSession, url: str):
    @asyncio.coroutine
    def download_multiple(session: aiohttp.ClientSession):
    urls = (
    'http://cnn.com',
    'http://nytimes.com',
    'http://google.com',
    'http://example.com',
    'http://leagueoflegends.com',
    'http://python.org',
    )
    download_futures = [download_file(session, url) for url in urls]
    print('Results')
  3. @harrisont harrisont revised this gist Jun 6, 2016. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions download_multiple.py
    View file
    Original file line number Diff line number Diff line change
    @@ -8,7 +8,7 @@ async def download_file(session: aiohttp.ClientSession, url: str):
    async with session.get(url) as response:
    assert response.status == 200
    # For large files use response.content.read(chunk_size) instead.
    return await response.read()
    return url, await response.read()


    @asyncio.coroutine
    @@ -21,15 +21,15 @@ def download_multiple(session: aiohttp.ClientSession):
    print('Results')
    for download_future in asyncio.as_completed(download_futures):
    result = yield from download_future
    print(result)
    return 'hello'
    print('finished:', result)
    return urls


    def main():
    with closing(asyncio.get_event_loop()) as loop:
    with aiohttp.ClientSession() as session:
    result = loop.run_until_complete(download_multiple(session))
    print(result)
    print('finished:', result)


    main()
  4. @harrisont harrisont revised this gist Jun 6, 2016. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions download_multiple.py
    View file
    Original file line number Diff line number Diff line change
    @@ -31,4 +31,5 @@ def main():
    result = loop.run_until_complete(download_multiple(session))
    print(result)


    main()
  5. @harrisont harrisont created this gist Jun 6, 2016.
    34 changes: 34 additions & 0 deletions download_multiple.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,34 @@
    import asyncio
    from contextlib import closing

    import aiohttp


    async def download_file(session: aiohttp.ClientSession, url: str):
    async with session.get(url) as response:
    assert response.status == 200
    # For large files use response.content.read(chunk_size) instead.
    return await response.read()


    @asyncio.coroutine
    def download_multiple(session: aiohttp.ClientSession):
    urls = (
    'http://google.com',
    'http://example.com',
    )
    download_futures = [download_file(session, url) for url in urls]
    print('Results')
    for download_future in asyncio.as_completed(download_futures):
    result = yield from download_future
    print(result)
    return 'hello'


    def main():
    with closing(asyncio.get_event_loop()) as loop:
    with aiohttp.ClientSession() as session:
    result = loop.run_until_complete(download_multiple(session))
    print(result)

    main()